Merge remote-tracking branch 'origin/main' into litellm_responses_drop_malformed_values

# Conflicts:
#	litellm/llms/openai/responses/count_tokens/token_counter.py
This commit is contained in:
mateo-berri 2026-10-29 19:38:57 -07:00
commit 814eb8c3fc
362 changed files with 21478 additions and 2860 deletions

View file

@ -1,14 +1,14 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "${GITHUB_ACTIONS:-}" = true ]; then
echo "Integration contracts are owned by CircleCI" >&2
exit 1
fi
suite="${1:?integration suite required}"
mode="${2:-standard}"
side="${3:-}"
requested_workers="${INTEGRATION_WORKERS:-}"
license_env=()
if [ -n "${LITELLM_LICENSE:-}" ]; then
license_env=("LITELLM_LICENSE=$LITELLM_LICENSE")
fi
if [ "$mode" = replica ]; then
results="test-results/integration-${suite}-replica"
elif [ "$mode" = parity ]; then
@ -84,7 +84,7 @@ if [ "$suite" = browser ]; then
export LITELLM_UI_PATH="$PWD/ui/litellm-dashboard/out"
test -f "$LITELLM_UI_PATH/index.html"
fi
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256((os.environ.get("CIRCLE_SHA1", "local") + os.environ.get("CIRCLE_WORKFLOW_ID", "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_SEED="$(.venv/bin/python -c 'import hashlib,os; print(int(hashlib.sha256(((os.environ.get("CIRCLE_SHA1") or os.environ.get("GITHUB_SHA") or "local") + (os.environ.get("CIRCLE_WORKFLOW_ID") or os.environ.get("GITHUB_RUN_ID") or "local")).encode()).hexdigest()[:8],16))')"
export INTEGRATION_ORDER_SEED="$INTEGRATION_SEED"
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma > "$results/prisma-generate.log" 2>&1
@ -107,7 +107,7 @@ sudo iptables -N integration_only
guard_created=true
sudo iptables -A integration_only -o lo -j ACCEPT
sudo iptables -A integration_only -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
for service in postgres-db redis-cache; do
for service in "${INTEGRATION_POSTGRES_CONTAINER:-postgres-db}" "${INTEGRATION_REDIS_CONTAINER:-redis-cache}"; do
address="$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$service")"
sudo iptables -A integration_only -d "$address" -j ACCEPT
done
@ -145,6 +145,9 @@ fi
if [ "$suite" = mcp ]; then
export INTEGRATION_WORKERS=2 INTEGRATION_COVERAGE=1
fi
if [ -n "$requested_workers" ]; then
export INTEGRATION_WORKERS="$requested_workers"
fi
isolated_workers=false
if [ "${INTEGRATION_WORKERS:-1}" -gt 1 ]; then
isolated_workers=true
@ -185,7 +188,7 @@ start_proxy() {
"${database_env[@]}" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True LITELLM_ENABLE_MCP_STDIO=true "${cost_map_env[@]}" \
AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 COVERAGE_FILE="$coverage_data" \
"${proxy_command[@]}" --config tests/integration/proxy_config.yaml \
@ -257,7 +260,7 @@ env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
"${shared_services[@]}" "${worker_coverage[@]}" \
INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
LITELLM_MODE=PRODUCTION \
LITELLM_LICENSE="${LITELLM_LICENSE:-}" \
"${license_env[@]}" \
INTEGRATION_SEED="$INTEGRATION_SEED" \
INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \
INTEGRATION_CASE_SHARDS="${CIRCLE_NODE_TOTAL:-1}" INTEGRATION_CASE_SHARD_INDEX="${CIRCLE_NODE_INDEX:-0}" \

119
.github/scripts/assert_secret_absent.py vendored Normal file
View file

@ -0,0 +1,119 @@
import base64
import io
import os
import sys
import urllib.parse
import zipfile
from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass
from functools import reduce
from itertools import chain
from pathlib import Path
from types import MappingProxyType
from typing import Final, TextIO
SECRET_VARIABLE: Final = "SCANNED_SECRET"
MIN_ENCODED_LENGTH: Final = 12
MAX_ARCHIVE_DEPTH: Final = 3
MASK: Final = "***"
Forms = tuple[tuple[str, str], ...]
@dataclass(frozen=True, slots=True)
class Hit:
location: str
form: str
def _aligned_encodings(raw: bytes, name: str, encode: Callable[[bytes], bytes]) -> Iterator[tuple[str, str]]:
for offset in range(3):
chunk: Final = raw[offset:]
yield f"{name} offset {offset}", encode(chunk[: len(chunk) - len(chunk) % 3]).decode()
def secret_forms(secret: str) -> Forms:
encoded: Final = (
("url-encoded", urllib.parse.quote(secret, safe="")),
*_aligned_encodings(secret.encode(), "base64", base64.b64encode),
*_aligned_encodings(secret.encode(), "base64url", base64.urlsafe_b64encode),
)
by_value: Final = MappingProxyType(
{value: name for name, value in encoded if value != secret and len(value) >= MIN_ENCODED_LENGTH}
)
return (("raw", secret), *((name, value) for value, name in by_value.items()))
def mask(text: str, forms: Forms) -> str:
return reduce(lambda masked, form: masked.replace(form[1], MASK), forms, text)
def _name_hits(location: str, name: str, forms: Forms) -> Iterator[Hit]:
for form, value in forms:
if value in name:
yield Hit(mask(location, forms), f"{form} in name")
def _content_hits(location: str, data: bytes, forms: Forms, depth: int) -> Iterator[Hit]:
for form, value in forms:
if value.encode() in data:
yield Hit(mask(location, forms), form)
if depth >= MAX_ARCHIVE_DEPTH or not zipfile.is_zipfile(io.BytesIO(data)):
return
with zipfile.ZipFile(io.BytesIO(data)) as archive:
for member in archive.infolist():
if member.is_dir():
continue
yield from _content_hits(f"{location}!{member.filename}", archive.read(member), forms, depth + 1)
def _walk(root: Path) -> Iterator[tuple[Path, bool]]:
for directory, subdirectories, files in os.walk(root):
yield from ((Path(directory, name), True) for name in subdirectories if Path(directory, name).is_symlink())
yield from ((Path(directory, name), False) for name in files)
def _path_hits(root: Path, path: Path, linked_directory: bool, forms: Forms) -> Iterator[Hit]:
location: Final = str(path.relative_to(root))
if linked_directory:
yield Hit(mask(location, forms), "directory symlink, not scanned")
return
yield from _name_hits(location, location, forms)
yield from _content_hits(location, path.read_bytes(), forms, 0)
def scan(root: Path, secret: str) -> tuple[int, tuple[Hit, ...]]:
forms: Final = secret_forms(secret)
entries: Final = tuple(_walk(root))
hits: Final = tuple(chain.from_iterable(_path_hits(root, path, linked, forms) for path, linked in entries))
return len(entries), hits
def main(argv: tuple[str, ...], environ: Mapping[str, str], out: TextIO) -> int:
if len(argv) != 1:
print(f"usage: assert_secret_absent.py <directory>, with the secret in ${SECRET_VARIABLE}", file=out)
return 2
secret: Final = environ.get(SECRET_VARIABLE, "")
root: Final = Path(argv[0])
if not secret:
print(f"{SECRET_VARIABLE} is empty, nothing to scan for", file=out)
return 0
if not root.is_dir():
print(f"{root} is not a directory, nothing to scan", file=out)
return 0
try:
scanned, hits = scan(root, secret)
except Exception as error:
print(f"scan failed with {type(error).__name__}, treating the results as unsafe", file=out)
return 1
for hit in hits:
print(f"secret found: {hit.location} ({hit.form})", file=out)
if hits:
print(f"{len(hits)} copies of the secret under {root}, refusing to upload", file=out)
return 1
print(f"no copy of the secret in {scanned} files under {root}", file=out)
return 0
if __name__ == "__main__":
sys.exit(main(tuple(sys.argv[1:]), os.environ, sys.stdout))

View file

@ -8,6 +8,7 @@ on:
permissions:
contents: read
pull-requests: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
@ -21,16 +22,19 @@ jobs:
with:
ref: ${{ github.event.pull_request.base.sha }}
persist-credentials: false
- name: Fetch the pull request head, its merge base, and the base branch tip
- name: Check out the base branch tip and fetch the pull request head and its merge base
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name)" || base_ref="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)"
target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${base_ref}" --jq '.commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
git checkout -q --detach "$target"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
echo "target=$target" >> "$GITHUB_OUTPUT"
- name: Set up uv

View file

@ -19,17 +19,17 @@ jobs:
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ github.event.pull_request.base.sha }}
ref: ${{ github.event.pull_request.base.ref }}
persist-credentials: false
- name: Fetch the pull request head, its merge base and its reviews
id: revisions
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
BASE_REF: ${{ github.event.pull_request.base.ref }}
run: |
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/reviews" --paginate --slurp | jq 'add // []' > "$RUNNER_TEMP/reviews.json"
echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"

199
.github/workflows/test-integration.yml vendored Normal file
View file

@ -0,0 +1,199 @@
name: integration
on:
pull_request:
types: [labeled]
schedule:
- cron: "5 8-20/3 * * 1-6"
timezone: America/Los_Angeles
workflow_dispatch:
inputs:
suite:
description: Integration group to run
type: choice
options:
- all
- providers
- extensions
- accounting
- management
- management-replica
- database
- database-replica
- mcp
- security
- cost
- sdk
- browser
default: all
workers:
description: Workers per group, blank keeps each group's default
type: string
default: ""
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name }}-${{ inputs.suite }}-${{ inputs.workers }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
integration:
name: integration ${{ matrix.suite }}
if: github.event_name != 'pull_request' || github.event.label.name == 'run-ci'
environment: integration-license
runs-on: ${{ fromJSON('{"providers":"ubuntu-latest-64-cores","extensions":"ubuntu-latest-64-cores","accounting":"ubuntu-latest-32-cores","management":"ubuntu-latest-32-cores","management-replica":"ubuntu-latest-32-cores"}')[matrix.suite] || 'ubuntu-latest-16-cores' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
suite: ${{ fromJSON((inputs.suite || 'all') == 'all' && '["providers","extensions","accounting","management","management-replica","database","database-replica","mcp","security","cost","sdk","browser"]' || format('["{0}"]', inputs.suite)) }}
env:
UV_PYTHON: "3.12"
INTEGRATION_WORKERS: ${{ inputs.workers || fromJSON('{"providers":"32","extensions":"48","accounting":"16","management":"16","management-replica":"16","cost":"16","mcp":"8","security":"8","database":"4","database-replica":"4"}')[matrix.suite] || '' }}
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
command: -c max_connections=1000 -c shared_preload_libraries=pg_stat_statements -c pg_stat_statements.track=all -c pg_stat_statements.max=20000
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: circle_test
ports:
- 5432:5432
options: >-
--shm-size 1g
--health-cmd "pg_isready -U postgres"
--health-interval 5s
--health-timeout 5s
--health-retries 20
redis:
image: redis:7-alpine@sha256:7aec734b2bb298a1d769fd8729f13b8514a41bf90fcdd1f38ec52267fbaa8ee6
ports:
- 6379:6379
options: >-
--health-cmd "redis-cli ping"
--health-interval 5s
--health-timeout 5s
--health-retries 20
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ env.UV_PYTHON }}
- name: Set up uv
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
timeout-minutes: 5
uses: ./.github/actions/cache-prisma-binaries
- name: Install dependencies
timeout-minutes: 20
run: .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Install redis-server 7.2.16
timeout-minutes: 3
run: |
mkdir -p "$RUNNER_TEMP/redis-bin"
cid="$(docker create redis:7.2.16@sha256:0637954999d01b7c9ce9167db2da50656e2590d3b884f1c600c5f63bb6e6773c)"
docker cp "${cid}:/usr/local/bin/redis-server" "$RUNNER_TEMP/redis-bin/redis-server"
docker rm "$cid"
"$RUNNER_TEMP/redis-bin/redis-server" --version | grep -q 'v=7.2.16'
echo "$RUNNER_TEMP/redis-bin" >> "$GITHUB_PATH"
- name: Install MCP SDK1 peer
if: matrix.suite == 'mcp'
timeout-minutes: 5
run: |
uv venv --python 3.12 .venv-mcp-peer
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
echo "MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$GITHUB_ENV"
- name: Set up Node.js
if: matrix.suite == 'browser'
timeout-minutes: 5
uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
with:
node-version-file: ui/litellm-dashboard/.nvmrc
cache: "npm"
cache-dependency-path: |
ui/litellm-dashboard/package-lock.json
tests/e2e/ui/package-lock.json
- name: Install browser dependencies and build the dashboard
if: matrix.suite == 'browser'
timeout-minutes: 20
run: |
(cd ui/litellm-dashboard && npm ci)
(cd tests/e2e/ui && npm ci)
sudo env PATH="$PATH" DEBIAN_FRONTEND=noninteractive \
node tests/e2e/ui/node_modules/@playwright/test/cli.js install-deps chromium
node tests/e2e/ui/node_modules/@playwright/test/cli.js install chromium
(cd ui/litellm-dashboard && NEXT_TELEMETRY_DISABLED=1 npm run build)
- name: Run integration contracts
timeout-minutes: 45
env:
SUITE: ${{ matrix.suite }}
INTEGRATION_POSTGRES_CONTAINER: ${{ job.services.postgres.id }}
INTEGRATION_REDIS_CONTAINER: ${{ job.services.redis.id }}
LITELLM_LICENSE: ${{ secrets.LITELLM_LICENSE }}
run: |
case "$SUITE" in
*-replica) bash .circleci/scripts/run_integration.sh "${SUITE%-replica}" replica ;;
*) bash .circleci/scripts/run_integration.sh "$SUITE" ;;
esac
- name: Verify every case ran exactly once
if: ${{ !cancelled() && matrix.suite != 'browser' }}
timeout-minutes: 2
run: |
record="$(find test-results -mindepth 2 -maxdepth 2 -name execution.json -print -quit)"
test -n "$record"
mkdir -p "$RUNNER_TEMP/shards/0"
cp "$record" "$RUNNER_TEMP/shards/0/execution.json"
.venv/bin/python -m tests.integration._support.sharding "$RUNNER_TEMP/shards" --shards 1
- name: Report MCP coverage
if: ${{ !cancelled() && matrix.suite == 'mcp' }}
timeout-minutes: 5
run: |
coverage_dir=test-results/integration-mcp/coverage
test -f "$coverage_dir/coverage.txt" && exit 0
export COVERAGE_FILE="$PWD/$coverage_dir/data"
.venv/bin/python -m coverage combine --rcfile=tests/integration/mcp_coverage.toml
.venv/bin/python -m coverage report --rcfile=tests/integration/mcp_coverage.toml > "$coverage_dir/coverage.txt"
.venv/bin/python -m coverage html --rcfile=tests/integration/mcp_coverage.toml -d "$coverage_dir/html"
tail -n 1 "$coverage_dir/coverage.txt"
- name: Check results for the license
id: secret-scan
if: ${{ !cancelled() }}
timeout-minutes: 5
env:
SCANNED_SECRET: ${{ secrets.LITELLM_LICENSE }}
run: python .github/scripts/assert_secret_absent.py test-results
- name: Upload results
if: ${{ !cancelled() && steps.secret-scan.outcome == 'success' }}
timeout-minutes: 10
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: integration-${{ matrix.suite }}-${{ github.run_id }}-${{ github.run_attempt }}
path: test-results/
retention-days: 14

View file

@ -48,11 +48,13 @@ jobs:
if: steps.changes.outputs.decision != 'skip'
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
BASE_REF=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || BASE_REF=$(retry gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
MERGE_BASE=$(retry gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_REF}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$MERGE_BASE"
retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
@ -264,6 +266,7 @@ jobs:
name: ui
permissions:
contents: read
pull-requests: read
runs-on: ubuntu-latest
timeout-minutes: 8
defaults:
@ -281,16 +284,12 @@ jobs:
id: changed
env:
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# base.sha is the base branch tip from when the PR was opened, while
# actions/checkout leaves HEAD on a merge of the PR into the *current*
# base tip. "$BASE_SHA"...HEAD therefore spans every base-branch commit
# landed since, so a PR that touches no UI file still gets linted
# against hundreds of other people's files. Diff the PR head against its
# own merge base instead, which is exactly what this PR changed.
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
: > "$RUNNER_TEMP/prettier_files.txt"
@ -630,6 +629,12 @@ jobs:
- name: check_migrations_lock_timeout
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
- name: check_no_none_string_error_fields
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_none_string_error_fields.py
- name: test_check_no_none_string_error_fields
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_check_no_none_string_error_fields.py
- name: check_no_publicly_known_master_key
run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py

View file

@ -245,6 +245,7 @@ jobs:
tests/unit/llms/chatgpt
tests/unit/llms/clarifai
tests/unit/llms/claude_code
tests/unit/llms/clf_ai_gateway
tests/unit/llms/cloudflare
tests/unit/llms/codestral
tests/unit/llms/codex
@ -1003,9 +1004,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1150,18 +1151,20 @@ jobs:
env:
CI: "true"
GH_TOKEN: ${{ github.token }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
full_suite() { npm run test -- --run --pool forks --maxWorkers=14; }
if [ -z "$BASE_SHA" ]; then
if [ -z "$PR_NUMBER" ]; then
echo "Push to $GITHUB_REF_NAME: running the full suite"
full_suite
exit 0
fi
merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
test -n "$merge_base"
git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
changed_files=()
@ -1451,9 +1454,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

View file

@ -153,7 +153,7 @@ lint-install:
# the same set CI's ':(glob)litellm/**/*.py' selects.
lint-format-check-changed: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
@base_ref=$$($(RESOLVE_BASE)) && \
changed=$$(git diff --name-only --diff-filter=ACMR "$$base_ref...HEAD" -- 'litellm/*.py') && \
changed=$$(git diff --name-only --diff-filter=ACMR --merge-base "$$base_ref" -- 'litellm/*.py') && \
files=$$(printf '%s\n' "$$changed" | grep -v '^litellm/enterprise/' || true) || exit $$?; \
if [ -z "$$files" ]; then \
echo "No changed litellm Python files to format-check."; \

View file

@ -379,6 +379,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Bytez (`bytez`)](https://docs.litellm.ai/docs/providers/bytez) | ✅ | ✅ | ✅ | | | | | | | |
| [Cerebras (`cerebras`)](https://docs.litellm.ai/docs/providers/cerebras) | ✅ | ✅ | ✅ | | | | | | | |
| [Clarifai (`clarifai`)](https://docs.litellm.ai/docs/providers/clarifai) | ✅ | ✅ | ✅ | | | | | | | |
| [CLF AI Gateway (`clf_ai_gateway`)](https://docs.litellm.ai/docs/providers/clf_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | |
| [Cloudflare AI Workers (`cloudflare`)](https://docs.litellm.ai/docs/providers/cloudflare_workers) | ✅ | ✅ | ✅ | | | | | | | |
| [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Cognition (`cognition`)](https://docs.litellm.ai/docs/providers/cognition) | ✅ | ✅ | ✅ | | | | | | | |
@ -398,6 +399,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | |
| [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ | | | ✅ | ✅ | | | |
| [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ | | | | | | | |
| [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) | | | | | | | | | | |
| [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ | | ✅ | | | | | |
| [Featherless AI (`featherless_ai`)](https://docs.litellm.ai/docs/providers/featherless_ai) | ✅ | ✅ | ✅ | | | | | | | |
| [Fireworks AI (`fireworks_ai`)](https://docs.litellm.ai/docs/providers/fireworks_ai) | ✅ | ✅ | ✅ | | | | | | | |

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.80"
version = "0.1.81"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.80"
version = "0.1.81"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

View file

@ -142,6 +142,7 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
"/test",
"/debug/memory/summary",
"/api/event_logging/batch",
"/eu_jev/v1/systemone",
}
)

File diff suppressed because one or more lines are too long

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.111"
version = "0.4.112"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.111"
version = "0.4.112"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -238,7 +238,8 @@ impl AnthropicBeta {
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
| Self::ThinkingBindingControls20260801
| Self::ThinkingDisplayUpdates20260818 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -275,7 +276,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,
BetaProvider::AzureAi,

View file

@ -633,6 +633,7 @@ fireworks_ai_models: Set = set()
baseten_models: Set[str] = set()
fireworks_ai_embedding_models: Set = set()
deepinfra_models: Set = set()
clf_ai_gateway_models: set[str] = set() # mutable-ok: filled at import like sibling sets
perplexity_models: Set = set()
watsonx_models: Set = set()
gemini_models: Set = set()
@ -859,6 +860,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
bedrock_converse_models.add(key)
elif value.get("litellm_provider") == "deepinfra":
deepinfra_models.add(key)
elif value.get("litellm_provider") == "clf_ai_gateway": # pyright: ignore[reportUnknownMemberType] # raw cost map
clf_ai_gateway_models.add(key) # pyright: ignore[reportUnknownArgumentType] # raw cost map key
elif value.get("litellm_provider") == "perplexity":
perplexity_models.add(key)
elif value.get("litellm_provider") == "watsonx":
@ -1110,6 +1113,7 @@ model_list = list(
| set(ollama_models)
| bedrock_models
| deepinfra_models
| clf_ai_gateway_models
| perplexity_models
| set(maritalk_models)
| runwayml_models
@ -1235,6 +1239,7 @@ def _build_models_by_provider() -> dict:
"ollama": ollama_models,
"ollama_chat": ollama_models,
"deepinfra": deepinfra_models,
"clf_ai_gateway": clf_ai_gateway_models,
"perplexity": perplexity_models,
"maritalk": maritalk_models,
"watsonx": watsonx_models,
@ -1783,6 +1788,9 @@ if TYPE_CHECKING:
from .llms.bespoke.decisions.transformation import (
BespokeDecisionsConfig as BespokeDecisionsConfig,
)
from .llms.eu_jev.decisions.transformation import (
EuJevDecisionsConfig as EuJevDecisionsConfig,
)
from .llms.openai.decisions.transformation import (
OpenAIDecisionsConfig as OpenAIDecisionsConfig,
)
@ -2108,6 +2116,9 @@ if TYPE_CHECKING:
LiteLLMProxyChatConfig as _LiteLLMProxyChatConfig,
)
from .llms.deepinfra.chat.transformation import DeepInfraConfig as _DeepInfraConfig
from .llms.clf_ai_gateway.chat.transformation import (
ClfAiGatewayConfig as _ClfAiGatewayConfig,
)
from .llms.llamafile.chat.transformation import (
LlamafileChatConfig as _LlamafileChatConfig,
)
@ -2137,6 +2148,7 @@ if TYPE_CHECKING:
IBMWatsonXAIConfig: Type[_IBMWatsonXAIConfig]
LiteLLMProxyChatConfig: Type[_LiteLLMProxyChatConfig]
DeepInfraConfig: Type[_DeepInfraConfig]
ClfAiGatewayConfig: type[_ClfAiGatewayConfig]
LlamafileChatConfig: Type[_LlamafileChatConfig]
LMStudioChatConfig: Type[_LMStudioChatConfig]
LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]

View file

@ -174,6 +174,7 @@ LLM_CONFIG_NAMES: Final = (
"OllamaDecisionsConfig",
"LayaDecisionsConfig",
"BespokeDecisionsConfig",
"EuJevDecisionsConfig",
"OpenAIDecisionsConfig",
"AzureDecisionsConfig",
"AzureAIDecisionsConfig",
@ -244,6 +245,7 @@ LLM_CONFIG_NAMES: Final = (
"MistralEmbeddingConfig",
"OpenAIImageVariationConfig",
"DeepInfraConfig",
"ClfAiGatewayConfig",
"DeepgramAudioTranscriptionConfig",
"TopazImageVariationConfig",
"OpenAITextCompletionConfig",
@ -753,6 +755,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
"LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
"BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
"EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
"OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
"AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),
"AzureAIDecisionsConfig": (".llms.azure_ai.decisions.transformation", "AzureAIDecisionsConfig"),
@ -973,6 +976,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"OpenAIImageVariationConfig",
),
"DeepInfraConfig": (".llms.deepinfra.chat.transformation", "DeepInfraConfig"),
"ClfAiGatewayConfig": (
".llms.clf_ai_gateway.chat.transformation",
"ClfAiGatewayConfig",
),
"DeepgramAudioTranscriptionConfig": (
".llms.deepgram.audio_transcription.transformation",
"DeepgramAudioTranscriptionConfig",

View file

@ -28,9 +28,11 @@ from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.secret_redaction import (
REDACTED,
python_redact_string,
python_redact_structured_value,
redact_internal_details,
redact_structured_value,
)
from litellm.litellm_core_utils.secret_redaction import (
redact_string as redact_secret_string,
@ -140,6 +142,32 @@ def redact_internal_details_from_client_message(value: str) -> str:
return redact_internal_details(value)
_CLIENT_FIELDS: Final = TypeAdapter(dict[str, object])
_CLIENT_ITEMS: Final = TypeAdapter(list[object])
def _redacted_client_field(key: str | None, value: object) -> object:
if isinstance(value, str):
return redact_internal_details(redact_structured_value(key, value))
if isinstance(value, dict):
return {
child_key: _redacted_client_field(child_key, child)
for child_key, child in _CLIENT_FIELDS.validate_python(value).items()
}
if isinstance(value, (list, tuple)):
return [_redacted_client_field(key, child) for child in _CLIENT_ITEMS.validate_python(value)]
if not isinstance(value, (int, float)):
return value
text: Final = str(value)
return value if redact_structured_value(key, text) == text else REDACTED
def redact_internal_details_from_client_fields(fields: Mapping[str, object]) -> object:
if not _ENABLE_SECRET_REDACTION:
return fields
return _redacted_client_field(None, dict(fields))
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.

View file

@ -8,10 +8,9 @@ The following caching mechanisms are supported:
2. **RedisSemanticCache**
3. **QdrantSemanticCache**
4. **InMemoryCache**
5. **DiskCache**
6. **S3Cache**
7. **AzureBlobCache**
8. **DualCache** (updates both Redis and an in-memory cache simultaneously)
5. **S3Cache**
6. **AzureBlobCache**
7. **DualCache** (updates both Redis and an in-memory cache simultaneously)
## Folder Structure
@ -20,7 +19,6 @@ litellm/caching/
├── base_cache.py
├── caching.py
├── caching_handler.py
├── disk_cache.py
├── dual_cache.py
├── in_memory_cache.py
├── qdrant_semantic_cache.py

View file

@ -1,6 +1,5 @@
from .azure_blob_cache import AzureBlobCache
from .caching import Cache, LiteLLMCacheType
from .disk_cache import DiskCache
from .dual_cache import DualCache
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache

View file

@ -18,7 +18,7 @@ from contextlib import contextmanager
from enum import Enum
from typing import Any, Final, Literal, cast
from pydantic import BaseModel, TypeAdapter
from pydantic import BaseModel, ConfigDict, TypeAdapter
import litellm
from litellm._internal_context import current_service_target, service_target
@ -31,7 +31,6 @@ from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
from .azure_blob_cache import AzureBlobCache
from .base_cache import BaseCache
from .disk_cache import DiskCache
from .dual_cache import DualCache # noqa: F401 # re-exported, callers import DualCache from litellm.caching.caching
from .gcs_cache import GCSCache
from .in_memory_cache import InMemoryCache
@ -44,6 +43,9 @@ from .s3_cache import S3Cache
RESPONSE_CACHE_TARGET: Final = "llm_response"
_ALWAYS_KEYED_PROVIDER_PARAMS: Final = frozenset({"continuation_token"})
_DECODED_LITERAL: Final = TypeAdapter(object)
_REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
list[dict[str, object]], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
@contextmanager
@ -153,7 +155,7 @@ class Cache:
Initializes the cache based on the given type.
Args:
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic", "s3" or "disk". Defaults to "local".
type (str, optional): The type of cache to initialize. Can be "local", "redis", "redis-semantic", "qdrant-semantic" or "s3". Defaults to "local".
# Redis Cache Args
host (str, optional): The host address for the Redis cache. Required if type is "redis".
@ -174,9 +176,6 @@ class Cache:
semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
# Disk Cache Args
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
# S3 Cache Args
s3_bucket_name (str, optional): The bucket name for the s3 cache. Defaults to None.
s3_region_name (str, optional): The region name for the s3 cache. Defaults to None.
@ -211,7 +210,7 @@ class Cache:
if not redis_startup_nodes:
_env_cluster_nodes: Final = litellm.get_secret("REDIS_CLUSTER_NODES")
if _env_cluster_nodes is not None and isinstance(_env_cluster_nodes, str):
redis_startup_nodes = json.loads(_env_cluster_nodes)
redis_startup_nodes = _REDIS_STARTUP_NODES_ADAPTER.validate_python(json.loads(_env_cluster_nodes))
if redis_startup_nodes:
# Only pass GCP parameters if they are provided
@ -306,8 +305,11 @@ class Cache:
account_url=azure_account_url,
container=azure_blob_container,
)
elif type == LiteLLMCacheType.DISK:
self.cache = DiskCache(disk_cache_dir=disk_cache_dir)
else:
supported_types: Final = ", ".join(member.value for member in LiteLLMCacheType)
raise ValueError(f"Unsupported cache type {type!r}. Supported types: {supported_types}")
if disk_cache_dir is not None:
verbose_logger.warning("Ignoring disk_cache_dir, the disk cache backend was removed")
if "cache" not in litellm.input_callback:
litellm.input_callback.append("cache")
if "cache" not in litellm.success_callback:
@ -1044,7 +1046,7 @@ def enable_cache(
Enable cache with the specified configuration.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache to enable. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache to enable. Defaults to "local".
host (Optional[str]): The host address of the cache server. Defaults to None.
port (Optional[str]): The port number of the cache server. Defaults to None.
password (Optional[str]): The password for the cache server. Defaults to None.
@ -1056,7 +1058,7 @@ def enable_cache(
None
Raises:
None
ValueError: If no cache is set yet and the cache type is unsupported.
"""
print_verbose("LiteLLM: Enabling Cache")
if "cache" not in litellm.input_callback:
@ -1091,7 +1093,7 @@ def update_cache(
Update the cache for LiteLLM.
Args:
type (Optional[Literal["local", "redis", "s3", "disk"]]): The type of cache. Defaults to "local".
type (Optional[Literal["local", "redis", "s3"]]): The type of cache. Defaults to "local".
host (Optional[str]): The host of the cache. Defaults to None.
port (Optional[str]): The port of the cache. Defaults to None.
password (Optional[str]): The password for the cache. Defaults to None.

View file

@ -9,7 +9,7 @@ This file is a wrapper around caching.py
This class is used to handle caching logic specific for LLM API requests (completion / embedding / text_completion / transcription etc)
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache, DiskCache) based on what the user has setup
It utilizes the (RedisCache, s3Cache, RedisSemanticCache, QdrantSemanticCache, InMemoryCache) based on what the user has setup
In each method it will call the appropriate method from caching.py
"""
@ -830,6 +830,13 @@ class LLMCachingHandler:
if is_internal_call.get():
return
if logging_obj.defer_async_logging and not logging_obj.stream:
from litellm.utils import dispatch_success_logging
logging_obj.model_call_details["cache_hit"] = cache_hit # rebind-ok: capture cache status before deferring
dispatch_success_logging(logging_obj, cached_result, start_time, end_time, False, False)
return
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

View file

@ -1,92 +0,0 @@
import json
from typing import TYPE_CHECKING, Any, Final
from pydantic import TypeAdapter
from .base_cache import BaseCache
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
Span = _Span
else:
Span = Any
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
class DiskCache(BaseCache):
def __init__(self, disk_cache_dir: str | None = None):
try:
import diskcache as dc
except ModuleNotFoundError as e:
raise ModuleNotFoundError("Please install litellm with `litellm[caching]` to use disk caching.") from e
# if users don't provider one, use the default litellm cache
if disk_cache_dir is None:
self.disk_cache = dc.Cache(".litellm_cache")
else:
self.disk_cache = dc.Cache(disk_cache_dir)
def set_cache(self, key, value, **kwargs):
if "ttl" in kwargs:
self.disk_cache.set(key, value, expire=kwargs["ttl"])
else:
self.disk_cache.set(key, value)
async def async_set_cache(self, key, value, **kwargs):
self.set_cache(key=key, value=value, **kwargs)
async def async_set_cache_pipeline(self, cache_list, **kwargs):
for cache_key, cache_value in cache_list:
if "ttl" in kwargs:
self.set_cache(key=cache_key, value=cache_value, ttl=kwargs["ttl"])
else:
self.set_cache(key=cache_key, value=cache_value)
def get_cache(self, key, **kwargs):
original_cached_response: Final = self.disk_cache.get(key)
if original_cached_response:
try:
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
except Exception:
cached_response = original_cached_response
return cached_response
return None
def batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
def increment_cache(self, key, value: int, **kwargs) -> int:
with self.disk_cache.transact():
cached_value: Final = self.get_cache(key=key)
init_value: Final = cached_value if isinstance(cached_value, int) else 0
new_value: Final = init_value + value
self.set_cache(key, new_value, **kwargs)
return new_value
async def async_get_cache(self, key, **kwargs):
return self.get_cache(key=key, **kwargs)
async def async_batch_get_cache(self, keys: list, **kwargs):
return_val: Final = []
for k in keys:
val = self.get_cache(key=k, **kwargs)
return_val.append(val)
return return_val
async def async_increment(self, key, value: int, **kwargs) -> int:
return self.increment_cache(key=key, value=value, **kwargs)
def flush_cache(self):
self.disk_cache.clear()
async def disconnect(self):
pass
def delete_cache(self, key):
self.disk_cache.pop(key)

View file

@ -29,6 +29,9 @@ it does not belong to. Queued clients are therefore bucketed by what it takes to
close them, and each bucket is ordered by deadline, so a reap walks the entries
that are due rather than the whole queue.
A bucket whose loop has been collected is dropped, and its clients are left to the
collector, as they are when the queue is full.
The queue holds its clients weakly, so waiting out a grace window never keeps
alive anything the collector would have reclaimed first.
"""
@ -36,6 +39,7 @@ alive anything the collector would have reclaimed first.
import asyncio
import contextlib
import inspect
import itertools
import threading
import time
import weakref
@ -55,6 +59,9 @@ _CLOSABLE_ANYWHERE: Final = "closable-anywhere"
_CLOSABLE_ON_ANY_LOOP: Final = "closable-on-any-loop"
_BucketKey = str | int
_LOOP_TOKENS: Final[weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, int]] = weakref.WeakKeyDictionary()
_LOOP_TOKEN_COUNTER: Final = itertools.count()
_LOOP_TOKEN_LOCK: Final = threading.Lock()
@dataclass(frozen=True, slots=True)
@ -65,12 +72,12 @@ class _PendingClose:
collector would otherwise have reclaimed first.
``needs_loop`` is set for a client whose close is a coroutine; those can only
be closed from the event loop they were evicted on, recorded in ``loop_id``.
be closed from the event loop they were evicted on, recorded in ``loop_token``.
A client that closes synchronously carries neither constraint.
"""
client_ref: "weakref.ref[object]"
loop_id: int | None
loop_token: int | None
needs_loop: bool
close_after: float
@ -79,16 +86,24 @@ def _bucket_key(pending: _PendingClose) -> _BucketKey:
"""Which reaps can close this entry: any at all, any running a loop, or one loop's."""
if not pending.needs_loop:
return _CLOSABLE_ANYWHERE
if pending.loop_id is None:
if pending.loop_token is None:
return _CLOSABLE_ON_ANY_LOOP
return pending.loop_id
return pending.loop_token
def _running_loop_id() -> int | None:
def running_loop_token() -> int | None:
try:
return id(asyncio.get_running_loop())
loop: Final = asyncio.get_running_loop()
except RuntimeError:
return None
try:
token: Final[int | None] = _LOOP_TOKENS.get(loop)
if token is not None:
return token
with _LOOP_TOKEN_LOCK:
return _LOOP_TOKENS.setdefault(loop, next(_LOOP_TOKEN_COUNTER))
except TypeError:
return id(loop)
def _close_function(client: object) -> Callable[[], object] | None:
@ -186,6 +201,8 @@ class EvictedClientCloser:
self._clock = clock
self._owned: weakref.WeakSet[object] = weakref.WeakSet()
self._buckets: dict[_BucketKey, deque[_PendingClose]] = {} # mutable-ok: deadline-ordered queues
self._dead_loop_tokens: deque[int] = deque() # mutable-ok: loop finalizers append without taking the queue lock
self._watched_loop_tokens: set[int] = set() # mutable-ok: tracks tokens until their dead buckets are discarded
self._pending_count = 0
self._queue_lock = threading.Lock() # the cache is reachable from every worker thread's loop
self._close_tasks: set[asyncio.Task[None]] = set() # mutable-ok: strong refs to running closes
@ -216,13 +233,19 @@ class EvictedClientCloser:
close_fn: Final = _close_function(client)
if close_fn is None:
return
loop_token: Final = running_loop_token()
needs_loop: Final = inspect.iscoroutinefunction(close_fn)
if self._pending_count >= self._max_pending:
return
self._discard_dead_loops()
if self._pending_count >= self._max_pending:
return
if needs_loop and loop_token is not None:
self._watch_loop(loop_token)
self._enqueue(
_PendingClose(
client_ref=weakref.ref(client),
loop_id=_running_loop_id(),
needs_loop=inspect.iscoroutinefunction(close_fn),
loop_token=loop_token,
needs_loop=needs_loop,
close_after=self._clock() + self._grace_seconds,
)
)
@ -236,7 +259,7 @@ class EvictedClientCloser:
if not self._pending_count:
return
now: Final = self._clock()
for pending in self._take_due(_running_loop_id(), now):
for pending in self._take_due(running_loop_token(), now):
client = pending.client_ref()
if client is None:
continue
@ -249,6 +272,29 @@ class EvictedClientCloser:
def pending_count(self) -> int:
return self._pending_count
def _watch_loop(self, loop_token: int) -> None:
if loop_token in self._watched_loop_tokens:
return
try:
weakref.finalize(asyncio.get_running_loop(), self._dead_loop_tokens.append, loop_token)
except TypeError:
return
self._watched_loop_tokens.add(loop_token)
def _discard_dead_loops(self) -> None:
with self._queue_lock:
self._discard_dead_loops_locked()
def _discard_dead_loops_locked(self) -> None:
while self._dead_loop_tokens:
self._discard_dead_loop_token_locked(self._dead_loop_tokens.popleft())
def _discard_dead_loop_token_locked(self, loop_token: int) -> None:
self._watched_loop_tokens.discard(loop_token)
bucket: Final = self._buckets.pop(loop_token, None)
if bucket is not None:
self._pending_count -= len(bucket)
def _enqueue(self, pending: _PendingClose) -> None:
"""Append to the entry's bucket, dropping any dead entries it queues behind.
@ -257,6 +303,7 @@ class EvictedClientCloser:
the front rather than having to be searched for.
"""
with self._queue_lock:
self._discard_dead_loops_locked()
bucket: Final = self._buckets.setdefault(_bucket_key(pending), deque())
while bucket and bucket[0].client_ref() is None:
bucket.popleft()
@ -264,9 +311,12 @@ class EvictedClientCloser:
bucket.append(pending)
self._pending_count += 1
def _take_due(self, loop_id: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (_CLOSABLE_ANYWHERE,) if loop_id is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_id)
def _take_due(self, loop_token: int | None, now: float) -> tuple[_PendingClose, ...]:
buckets = (
(_CLOSABLE_ANYWHERE,) if loop_token is None else (_CLOSABLE_ANYWHERE, _CLOSABLE_ON_ANY_LOOP, loop_token)
)
with self._queue_lock:
self._discard_dead_loops_locked()
return tuple(pending for key in buckets for pending in self._drain_locked(key, now))
def _drain_locked(self, key: _BucketKey, now: float) -> Iterator[_PendingClose]:

View file

@ -2,10 +2,9 @@
Add the event loop to the cache key, to prevent event loop closed errors.
"""
import asyncio
from typing import Final
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer
from .evicted_client_closer import EvictedClientCloser, default_evicted_client_closer, running_loop_token
from .in_memory_cache import InMemoryCache
@ -43,24 +42,22 @@ class LLMClientCache(InMemoryCache):
self.evicted_client_closer.schedule(evicted)
self.evicted_client_closer.reap()
def update_cache_key_with_event_loop(self, key):
def update_cache_key_with_event_loop(self, key: str) -> str:
"""
Add the event loop to the cache key, to prevent event loop closed errors.
If none, use the key as is.
"""
try:
event_loop: Final = asyncio.get_running_loop()
stringified_event_loop: Final = str(id(event_loop))
return f"{key}-{stringified_event_loop}"
except RuntimeError: # handle no current running event loop
return key
token: Final = running_loop_token()
return f"{key}-{token}" if token is not None else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
def _scoped_key(self, key: str, loop_bound: bool) -> str:
return self.update_cache_key_with_event_loop(key) if loop_bound else key
def set_cache(self, key: str, value: object, litellm_owned_client: bool = False, loop_bound: bool = True, **kwargs):
"""``litellm_owned_client`` marks a client litellm built, so it may be closed once evicted."""
if litellm_owned_client:
self.evicted_client_closer.mark_owned(value)
key = self.update_cache_key_with_event_loop(key)
return super().set_cache(key, value, **kwargs)
return super().set_cache(self._scoped_key(key, loop_bound), value, **kwargs)
async def async_set_cache(self, key: str, value: object, litellm_owned_client: bool = False, **kwargs):
if litellm_owned_client:
@ -68,13 +65,13 @@ class LLMClientCache(InMemoryCache):
key = self.update_cache_key_with_event_loop(key)
return await super().async_set_cache(key, value, **kwargs)
def get_cache(self, key, **kwargs):
key = self.update_cache_key_with_event_loop(key)
def get_cache(self, key: str, loop_bound: bool = True, **kwargs):
scoped_key: Final = self._scoped_key(key, loop_bound)
self.evicted_client_closer.reap()
return super().get_cache(key, **kwargs)
return super().get_cache(scoped_key, **kwargs)
async def async_get_cache(self, key, **kwargs):
async def async_get_cache(self, key: str, **kwargs):
key = self.update_cache_key_with_event_loop(key)
return await super().async_get_cache(key, **kwargs)

View file

@ -1979,6 +1979,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
self._tool_call_index_map: dict[int, int] = {} # mutable-ok: per-stream accumulator state
self._structured_output: Final = structured_output
self._first_message_item_done: bool = False
self._reasoning_summary_streamed: bool = False
def _handle_string_chunk(
self, str_line: Union[str, "BaseModel"]
@ -2021,6 +2022,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map: dict[int, int] | None = None, # mutable-ok: per-stream state, remapped in place
replays_openai_history: bool = False,
structured_output: bool = False,
reasoning_summary_streamed: bool = False,
) -> "ModelResponseStream":
"""
Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@ -2030,6 +2032,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map: Per-stream output_index -> sequential tool_call index map
replays_openai_history: Record the completed output's item sequence for the next turn
structured_output: Only the first message item became content
reasoning_summary_streamed: Whether earlier chunks already streamed reasoning summary text
Returns:
ModelResponseStream: OpenAI-formatted streaming chunk
@ -2210,13 +2213,15 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
else:
raise ValueError(f"Chat provider: Invalid text delta {parsed_chunk}")
elif event_type == "response.reasoning_summary_part.added" and reasoning_summary_streamed:
return ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(reasoning_content=" "))])
elif event_type == "response.reasoning_summary_text.delta":
content_part = parsed_chunk.get("delta", None)
if content_part:
return ModelResponseStream(
choices=[
StreamingChoices(
index=cast(int, parsed_chunk.get("summary_index")),
index=0,
delta=Delta(reasoning_content=content_part),
)
]
@ -2317,10 +2322,13 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
tool_call_index_map=self._tool_call_index_map,
replays_openai_history=self._replays_openai_history,
structured_output=self._structured_output,
reasoning_summary_streamed=self._reasoning_summary_streamed,
)
)
if self._structured_output and _is_message_item_done(chunk):
self._first_message_item_done = True
if _responses_event_type(chunk) == "response.reasoning_summary_text.delta" and chunk.get("delta"):
self._reasoning_summary_streamed = True
return self._with_served_service_tier(self._with_stream_scoped_id(translated))
def _is_text_delta_after_structured_answer(self, chunk: Mapping[str, object]) -> bool:

View file

@ -753,7 +753,13 @@ ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
ANTHROPIC_IMAGE_MAX_LONG_EDGE_PX: Final = 1568
ANTHROPIC_IMAGE_MAX_PIXELS: Final = 1_150_000
ANTHROPIC_IMAGE_PIXELS_PER_TOKEN: Final = 750
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS: Final = 13
OPENAI_CHAT_PDF_FILE_TOKENS: Final = 192
PDF_DATA_URL_PREFIX: Final = "data:application/pdf;base64,"
PDF_MAX_FORM_XOBJECT_DEPTH: Final = 32
PDF_MAX_XOBJECT_DRAWS_PER_PAGE: Final = 10_000
OPENAI_MODEL_NAME_MAX_DASHES: Final = 16
OPENAI_CHAT_MODES: Final = frozenset({"chat", "responses"})
VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
"low": 1,
@ -813,6 +819,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
"ollama",
"ollama_chat",
"deepinfra",
"clf_ai_gateway",
"perplexity",
"mistral",
"groq",
@ -1012,6 +1019,7 @@ openai_compatible_endpoints: Final[list[str]] = [
"api.perplexity.ai",
"api.endpoints.anyscale.com/v1",
"api.deepinfra.com/v1/openai",
"api.clfaigateway.dev/v1",
"api.mistral.ai/v1",
"codestral.mistral.ai/v1/chat/completions",
"codestral.mistral.ai/v1/fim/completions",
@ -1083,6 +1091,7 @@ openai_compatible_providers: Final[list[str]] = [
"deepseek",
"tencent",
"deepinfra",
"clf_ai_gateway",
"perplexity",
"xinference",
"xai",

View file

@ -153,7 +153,9 @@ from litellm.utils import (
ProviderConfigManager,
TextCompletionResponse,
TranscriptionResponse,
cached_cost_map_key,
cached_get_model_info_helper,
mapped_model_fill_missing,
model_info_with_pricing_overrides,
runtime_model_cost_registrations,
token_counter,
@ -962,21 +964,17 @@ def select_model_name_for_cost_calc(
"""
1. If custom pricing is true, return received model name
2. If base_model is set (e.g. for azure models), return that
3. If completion response has model set return that
3. If completion response has model set return that, unless only the requested model's cost-map row has rates
4. Check if model is passed in return that
"""
return_model: str | None = None
custom_llm_provider = _get_provider_for_cost_calc(model=model, custom_llm_provider=custom_llm_provider)
completion_response_model: str | None = None
if completion_response is not None:
if isinstance(completion_response, BaseModel):
completion_response_model = getattr(completion_response, "model", None)
elif isinstance(completion_response, dict):
completion_response_model = completion_response.get("model", None)
completion_response_model: Final = _get_response_model(completion_response)
hidden_params: Final[dict | None] = getattr(completion_response, "_hidden_params", None)
provider_response_model: Final = _get_hidden_str_for_cost_calc(hidden_params, "provider_response_model")
requested_model: Final = model or _get_hidden_str_for_cost_calc(hidden_params, "litellm_model_name")
explicit_pricing: Final = custom_pricing is True or base_model is not None
configured_model: Final = model or (
completion_response_model if isinstance(completion_response_model, str) else None
@ -1002,35 +1000,72 @@ def select_model_name_for_cost_calc(
else:
return_model = model
elif base_model is not None or bedrock_pricing_model is not None or provider_response_model is not None:
return_model = base_model if base_model is not None else bedrock_pricing_model or provider_response_model
elif base_model is not None or bedrock_pricing_model is not None:
return_model = base_model if base_model is not None else bedrock_pricing_model
elif provider_response_model is not None:
return _served_or_priced_requested_model(
provider_response_model, requested_model, custom_llm_provider, priced_region
)
elif completion_response_model is None and hidden_params is not None:
if hidden_params.get("model", None) is not None and len(hidden_params["model"]) > 0:
return_model = hidden_params.get("model", model)
if return_model is None and completion_response_model is not None:
return_model = completion_response_model
return _served_or_priced_requested_model(
completion_response_model, requested_model, custom_llm_provider, priced_region
)
if return_model is None and model is not None:
return_model = model
if (
return_model is not None
and custom_llm_provider is not None
and not _model_contains_known_llm_provider(return_model)
): # add provider prefix if not already present, to match model_cost
provider_prefix: Final = (
custom_llm_provider if priced_region is None else f"{custom_llm_provider}/{priced_region}"
)
return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", priced_region)
return return_model
if return_model is None:
return None
return _cost_map_model_name(return_model, custom_llm_provider, priced_region)
_select_model_name_for_cost_calc = select_model_name_for_cost_calc
def _cost_map_model_name(model: str, custom_llm_provider: str | None, region_name: str | None) -> str:
if custom_llm_provider is None or _model_contains_known_llm_provider(model):
return model
provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
return _strip_unregistered_leading_segments(f"{provider_prefix}/{model}", region_name)
def _rule_declares_rates(model: str, custom_llm_provider: str | None) -> bool:
rule_info: Final = unmapped_model_generalization(model, custom_llm_provider)
return rule_info is not None and _cost_map_entry_prices_anything(rule_info)
def _resolves_to_rates(model: str, custom_llm_provider: str | None) -> bool:
key: Final = cached_cost_map_key(model, custom_llm_provider)
if key is None:
return False
entry: Final = _raw_cost_map_entry(key)
if entry is None:
return _rule_declares_rates(model, custom_llm_provider)
fill_missing: Final = mapped_model_fill_missing(key, entry)
return _cost_map_entry_prices_anything(entry if fill_missing is None else {**fill_missing, **entry})
def _served_or_priced_requested_model(
served_model: str, requested_model: str | None, custom_llm_provider: str | None, priced_region: str | None
) -> str:
served: Final = _cost_map_model_name(served_model, custom_llm_provider, priced_region)
if requested_model is None:
return served
requested: Final = _cost_map_model_name(requested_model, custom_llm_provider, None)
if served in (requested, requested.partition("/")[2]):
return served
pricing_provider: Final = _get_provider_for_cost_calc(requested, custom_llm_provider)
if _resolves_to_rates(served, pricing_provider):
return served
return requested if _resolves_to_rates(requested, pricing_provider) else served
def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
"""Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
@ -1887,7 +1922,10 @@ def completion_cost(
video_resolution=video_resolution,
)
elif call_type in _SPEECH_CALL_TYPES:
prompt_characters = litellm.utils.count_characters(text=prompt)
prompt_characters = _speech_billed_characters(
completion_response if isinstance(completion_response, HttpxBinaryResponseContent) else None,
prompt,
)
elif call_type in _TRANSCRIPTION_CALL_TYPES:
# Check _hidden_params first (duration stored there to
# avoid polluting the response body), then fall back to
@ -2366,6 +2404,11 @@ def response_cost_calculator(
raise e
def _speech_billed_characters(speech_response: HttpxBinaryResponseContent | None, prompt: str) -> int:
reported: Final = speech_response.hidden_params.get("billed_characters") if speech_response is not None else None
return reported if isinstance(reported, int) else litellm.utils.count_characters(text=prompt)
def _video_input_image_cost(
model: str,
custom_llm_provider: str | None,
@ -3508,7 +3551,7 @@ def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str)
lookups cannot strip that prefix, so a zero-rated override read as declaring nothing, and a
session that should bill nothing fell through to the public rates instead.
"""
resolved: Final = _get_model_info_or_none(model_name, custom_llm_provider)
resolved: Final = _cost_map_model_info(model_name, custom_llm_provider)
entries: Final = (
litellm.model_cost.get(resolved.get("key")) if resolved is not None else None,
litellm.model_cost.get(model_name),
@ -3704,9 +3747,9 @@ def handle_realtime_transcription_cost_calculation(
return 0.0
model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
model_info: Final = _get_model_info_or_none(model_name, custom_llm_provider)
model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
override_info: Final = (
_get_model_info_or_none(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
_cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
)
total_cost = 0.0
@ -3737,13 +3780,6 @@ def _get_transcription_model_name_from_results(
return None
def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo | None:
try:
return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
except Exception:
return None
_TranscriptionRateKey: TypeAlias = Literal[
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
]

View file

@ -53,7 +53,7 @@ from litellm.main import (
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.llms.openai import ImageGenerationRequestQuality
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import CustomPricingLiteLLMParams, GenericLiteLLMParams
from litellm.types.utils import (
LITELLM_IMAGE_VARIATION_PROVIDERS,
LlmProviders,
@ -1010,6 +1010,7 @@ def image_edit(
optional_params=dict(image_edit_request_params),
litellm_params={
**image_edit_request_params,
**litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True),
"litellm_call_id": litellm_call_id,
"model_info": model_info,
"vertex_location": litellm_params.vertex_location,

View file

@ -20,7 +20,7 @@ class ImageEditRequestUtils:
image_edit_optional_params: ImageEditOptionalRequestParams,
drop_params: bool | None = None,
additional_drop_params: Collection[str] | None = None,
) -> dict:
) -> dict[str, object]:
"""
Get optional parameters for the image edit API.

View file

@ -102,18 +102,14 @@ class AgentOps(OpenTelemetry):
"Connection": "keep-alive",
}
client: Final = get_httpx_client()
try:
response: Final = client.post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
response: Final = get_httpx_client().post(
url=auth_endpoint,
headers=headers,
json={"api_key": api_key},
timeout=10,
)
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
if response.status_code != 200:
raise Exception(f"Failed to fetch auth token: {response.text}")
return _DECODED_JSON.validate_python(response.json())
finally:
client.close()
return _DECODED_JSON.validate_python(response.json())

View file

@ -5,9 +5,17 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
import urllib.parse
from typing import Final
from pydantic import BaseModel, ConfigDict
from litellm.llms.custom_httpx.http_handler import HTTPHandler
class _PromptVersionResponse(BaseModel):
model_config = ConfigDict(extra="ignore", strict=True, frozen=True, hide_input_in_errors=True)
data: dict[str, object] | None = None
def _sanitize_id(identifier: str) -> str:
"""Reject path traversal characters and URL-encode the identifier."""
if any(c in identifier for c in ("/", "\\", "#", "?")):
@ -71,8 +79,7 @@ class ArizePhoenixClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final = response.json()
return data.get("data")
return _PromptVersionResponse.model_validate(response.json()).data
except Exception as e:
# Check if it's an HTTP error

View file

@ -35,6 +35,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy._types import UserAPIKeyAuth
Span = _Span
else:
Span = Any
@ -126,6 +128,26 @@ def _starts_its_own_trace(span: object, kwargs: Mapping[str, object]) -> bool:
)
def _a_destination_records_the_call(kwargs: Mapping[str, object]) -> bool:
"""Whether a key's or team's ``langfuse_otel`` destination gets this call from the OTel v2 fan-out, which records
every call but one whose logging payload shows it never reached a provider."""
from litellm.integrations.otel.model.metadata import LLMCallEvent
if not _a_destination_carries_the_trace():
return False
call: Final = LLMCallEvent.from_dict(kwargs)
return call.payload is None or call.reached_a_provider
def _a_destination_carries_the_trace() -> bool:
"""Whether this request resolved a key's or team's ``langfuse_otel`` destination, whose fan-out sends it the whole
trace."""
from litellm.integrations.otel.model.config import ExporterOwner
from litellm.integrations.otel.plumbing.context import destination_backends
return ExporterOwner.LANGFUSE_OTEL in destination_backends()
class LangfuseOtelLogger(OpenTelemetry):
def __init__(self, config=None, *args, **kwargs):
# Prevent LangfuseOtelLogger from modifying global environment variables by constructing config manually
@ -541,6 +563,27 @@ class LangfuseOtelLogger(OpenTelemetry):
"""
return None
def _emit_once(
self,
kwargs: dict[str, object], # mutable-ok: the v1 dedupe marker is written into the request's kwargs
*scope: object,
) -> bool:
"""Whether to emit this span: once per request, and never for a call a Langfuse destination records."""
return not _a_destination_records_the_call(kwargs) and super()._emit_once(kwargs, *scope) # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
async def async_post_call_failure_hook(
self,
request_data: dict[str, object], # mutable-ok: the v1 hook contract
original_exception: Exception,
user_api_key_dict: "UserAPIKeyAuth",
traceback_str: str | None = None,
) -> None:
"""Record the proxy failure of a request no Langfuse destination carries."""
if not _a_destination_carries_the_trace():
await super().async_post_call_failure_hook( # pyright: ignore[reportUnknownMemberType] # the v1 hook takes a bare dict
request_data, original_exception, user_api_key_dict, traceback_str
)
async def async_service_success_hook(self, *args, **kwargs):
"""
Langfuse should not receive service success logs.

View file

@ -72,7 +72,7 @@ if TYPE_CHECKING:
)
from litellm.proxy.proxy_server import UserAPIKeyAuth as _UserAPIKeyAuth
Span = _Span | Any
Span = _Span
Tracer = _Tracer
Context = _Context
SpanExporter = _SpanExporter
@ -1440,7 +1440,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
span = None
# Only set attributes if the span is still recording (not closed)
# Note: parent_span is guaranteed to be not None here
if hasattr(parent_span, "set_status"):
if parent_span is not None and hasattr(parent_span, "set_status"):
parent_span.set_status(Status(StatusCode.OK))
self.set_attributes(parent_span, kwargs, response_obj)
# Raw-request as direct child of parent_span
@ -2233,7 +2233,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# record error on parent span (keeps hierarchy shallow)
# Only set attributes if the span is still recording (not closed)
# Note: parent_otel_span is guaranteed to be not None here
if parent_otel_span.is_recording():
if parent_otel_span is not None and parent_otel_span.is_recording():
self.set_attributes(parent_otel_span, kwargs, response_obj)
parent_error_message: Final = self._record_exception_on_span(span=parent_otel_span, kwargs=kwargs)
parent_otel_span.set_status(Status(StatusCode.ERROR, parent_error_message))

View file

@ -13,9 +13,9 @@ from opentelemetry.trace.status import Status, StatusCode
from litellm.integrations.otel.mappers import resolve_mappers
from litellm.integrations.otel.mappers.base import AttributeMapper, AttrValue, SpanData
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES
from litellm.integrations.otel.mappers.langfuse import LANGFUSE_REQUEST_METADATA_ATTRIBUTES, LangfuseMapper
from litellm.integrations.otel.mappers.openinference import fit_indexed_messages
from litellm.integrations.otel.model.config import OpenTelemetryV2Config
from litellm.integrations.otel.model.config import ExporterOwner, OpenTelemetryV2Config
from litellm.integrations.otel.model.payloads import (
GuardrailSpanData,
LLMCallSpanData,
@ -34,6 +34,7 @@ from litellm.integrations.otel.model.spans import (
mcp_tool_call_span_name,
service_span_name,
)
from litellm.integrations.otel.plumbing.context import destination_backends
from litellm.integrations.otel.plumbing.events import GenAIEventRecorder
from litellm.integrations.otel.plumbing.providers import to_otel_span_kind
@ -168,6 +169,9 @@ class SpanEmitter:
self._mappers: list[AttributeMapper] = (
list(mappers) if mappers is not None else resolve_mappers(config.mapper_names)
)
self._langfuse: Final = (
None if any(isinstance(mapper, LangfuseMapper) for mapper in self._mappers) else LangfuseMapper()
)
# Bounded LRU (ordered by insertion / most-recent touch). Storing keys
# only — the value is unused — so it behaves like a capped set.
self._emitted: OrderedDict[tuple[str, SpanRole], None] = OrderedDict()
@ -295,17 +299,27 @@ class SpanEmitter:
)
else None
)
mapped: Final = MappingProxyType(
{key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
configured: Final = {key: value for mapper in self._mappers for key, value in mapper.map(data).items()}
for_langfuse: Final = (
dict(
sorted(
self._langfuse.map(data).items(),
key=lambda item: item[0] not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
)
)
if self._langfuse is not None and ExporterOwner.LANGFUSE_OTEL in destination_backends()
else _NO_ATTRIBUTES
)
mapped: Final = MappingProxyType({**configured, **for_langfuse})
stamped_later: Final = error_attributes(error) if error else _NO_ATTRIBUTES
reserved: Final = len(stamped_later.keys() - mapped.keys())
overwrites: Final = carried_keys(span) & mapped.keys()
# The Langfuse request metadata sheds first, so a span at its attribute limit loses nothing else to it.
# The Langfuse request metadata, and the Langfuse vocabulary only a destination asked for, shed first,
# so a span at its attribute limit loses nothing else to them.
fitted: Final = fit_indexed_messages(
MappingProxyType({key: value for key, value in mapped.items() if key not in overwrites}),
attribute_budget(span, reserved),
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
LANGFUSE_REQUEST_METADATA_ATTRIBUTES.union(for_langfuse),
)
for key, value in mapped.items():
if key in overwrites or key in fitted:

View file

@ -198,6 +198,7 @@ class OpenTelemetryV2(CustomLogger):
tracer_provider: TracerProvider | None = None,
logger_provider: LoggerProvider | None = None,
meter_provider: "MeterProvider | None" = None,
console_default: bool = False,
**kwargs: Unpack[_CustomLoggerOptions],
) -> None:
super().__init__(**kwargs)
@ -206,7 +207,7 @@ class OpenTelemetryV2(CustomLogger):
self._tracer_provider: TracerProvider = (
tracer_provider
if tracer_provider is not None
else build_tracer_provider(self.config, tenant_overrides=True)
else build_tracer_provider(self.config, tenant_overrides=True, console_default=console_default)
)
self.tracer: Tracer = get_tracer(self._tracer_provider, LITELLM_TRACER_NAME)
self._metrics_recorder = self._init_metrics(meter_provider)
@ -217,7 +218,9 @@ class OpenTelemetryV2(CustomLogger):
mappers=resolve_mappers(self.config.mapper_names),
event_recorder=self._init_events(logger_provider),
)
self._tenant_tracers = TenantTracerCache(self.config, callback_name, LITELLM_TRACER_NAME)
self._tenant_tracers = TenantTracerCache(
self.config, callback_name, LITELLM_TRACER_NAME, console_default=console_default
)
self._open_llm_calls: OrderedDict[str, _LLMCallSpan] = OrderedDict()
self._init_otel_logger_on_litellm_proxy()
@ -541,9 +544,7 @@ class OpenTelemetryV2(CustomLogger):
# gate rejection carries ``is_no_upstream_call`` and gets no span. A cache
# hit never hands off to a provider, so ``cache_hit`` substitutes for the
# upstream stamp, since the payload proves the call completed.
if carrier is None and (
call.is_no_upstream_call or call.payload is None or (not call.upstream_started and not call.cache_hit)
):
if carrier is None and not call.reached_a_provider:
return None
try:
return self._finish_carrier(carrier, call, start_time, end_time)
@ -1091,6 +1092,7 @@ def build_otel_v2_logger(
logger_provider: LoggerProvider | None = None,
meter_provider: "MeterProvider | None" = None,
settings: Mapping[str, object] = MappingProxyType({}),
console_default: bool = False,
) -> OpenTelemetryV2:
return _logger_class(config)(
config=config,
@ -1098,6 +1100,7 @@ def build_otel_v2_logger(
tracer_provider=tracer_provider,
logger_provider=logger_provider,
meter_provider=meter_provider,
console_default=console_default,
**settings,
)

View file

@ -103,6 +103,18 @@ class LangfuseMapper:
),
}
#: Every attribute this mapper can write on a model call.
LLM_CALL_ATTRIBUTES: Final[frozenset[str]] = frozenset(
{
*_LLM_CALL_ATTRS,
*_BLOB_ATTRS,
LANGFUSE_TRACE_NAME,
LANGFUSE_TRACE_USER_ID,
LANGFUSE_TRACE_SESSION_ID,
LANGFUSE_TRACE_TAGS,
}
)
def map(self, data: SpanData) -> AttributeMap:
match data:
case LLMCallSpanData():

View file

@ -121,6 +121,17 @@ class ExporterSpec(LiteLLMBaseModel):
)
def is_unconfigured_placeholder(spec: ExporterSpec) -> bool:
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
No field set is what says the operator asked for nothing: an exporter they did
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
and so does the gated spec ``credential_gated_exporters`` appends, which would
otherwise eat itself when one preset layers onto another.
"""
return not spec.model_fields_set
class _EnvWithoutBareExcludedServices(PydanticBaseSettingsSource):
def __init__(self, settings_cls: type[BaseSettings], env_settings: PydanticBaseSettingsSource) -> None:
super().__init__(settings_cls)

View file

@ -272,6 +272,11 @@ class LLMCallEvent:
trace: TraceControls
session_id: str | None
@property
def reached_a_provider(self) -> bool:
"""Whether the call was handed to a provider or answered from the response cache."""
return not self.is_no_upstream_call and self.payload is not None and (self.upstream_started or self.cache_hit)
@classmethod
def from_dict(cls, kwargs: Mapping[str, object]) -> LLMCallEvent:
raw_payload: Final = kwargs.get("standard_logging_object")

View file

@ -47,6 +47,7 @@ from litellm.integrations.otel.mappers.langfuse import (
LANGFUSE_OBSERVATION_OUTPUT,
LANGFUSE_REQUEST_METADATA_ATTRIBUTES,
LANGFUSE_TRACE_NAME,
LangfuseMapper,
)
from litellm.integrations.otel.model.config import (
CaptureMessageContent,
@ -55,6 +56,7 @@ from litellm.integrations.otel.model.config import (
OpenTelemetryV2Config,
capture_message_content_from_env,
captures_span_content,
is_unconfigured_placeholder,
)
from litellm.integrations.otel.model.semconv import (
DB,
@ -490,19 +492,25 @@ def _is_litellm_span(span: ReadableSpan) -> bool:
return scope is not None and scope.name == LITELLM_TRACER_NAME
def without_langfuse_request_metadata(span: ReadableSpan) -> ReadableSpan:
"""``span`` without the request metadata the Langfuse mapper writes for a Langfuse destination.
def without_destination_langfuse_attributes(span: ReadableSpan, langfuse_mapped: bool) -> ReadableSpan:
"""``span`` without the Langfuse attributes only a Langfuse destination asked for.
Only LiteLLM's own spans lose it: any other span keeps every key its own code set.
That is the request metadata the Langfuse mapper writes and, on a model call written by
a logger whose own mappers leave the Langfuse vocabulary out (``langfuse_mapped`` off),
every Langfuse attribute, since that logger wrote them for a key's or team's Langfuse
destination alone. Only LiteLLM's own spans lose them: any other span keeps every key
its own code set.
"""
attributes: Final = span.attributes or _NO_ATTRIBUTES
if not (_is_litellm_span(span) and any(key in LANGFUSE_REQUEST_METADATA_ATTRIBUTES for key in attributes)):
dropped: Final = (
LANGFUSE_REQUEST_METADATA_ATTRIBUTES
if langfuse_mapped or not is_llm_call_span(span)
else LangfuseMapper.LLM_CALL_ATTRIBUTES
)
if not (_is_litellm_span(span) and any(key in dropped for key in attributes)):
return span
return _CountedSpanView(
span,
MappingProxyType(
{key: value for key, value in attributes.items() if key not in LANGFUSE_REQUEST_METADATA_ATTRIBUTES}
),
span, MappingProxyType({key: value for key, value in attributes.items() if key not in dropped})
)
@ -658,6 +666,7 @@ def _for_destination(
span: ReadableSpan,
destination: "OtelDestination",
default_capture: str,
langfuse_mapped: bool,
) -> ReadableSpan:
"""The view of ``span`` a tenant destination receives.
@ -672,13 +681,16 @@ def _for_destination(
and the query string of the request URL, which can hold the same key. A
destination whose effective setting disables span content loses the prompt and
response content too, and a destination for any backend but Langfuse loses the
request metadata the Langfuse mapper writes on LiteLLM's own spans. The span itself
stays, so the tenant still gets the whole trace tree.
Langfuse attributes only a Langfuse destination asked for (see
:func:`without_destination_langfuse_attributes`). The span itself stays, so the
tenant still gets the whole trace tree.
"""
extra: Final = destination.resource_attributes
defaults: Final = destination.resource_defaults
source: Final = (
span if destination.callback_name == ExporterOwner.LANGFUSE_OTEL else without_langfuse_request_metadata(span)
span
if destination.callback_name == ExporterOwner.LANGFUSE_OTEL
else without_destination_langfuse_attributes(span, langfuse_mapped)
)
attributes: Final = source.attributes or _NO_ATTRIBUTES
database: Final = _is_database_span(attributes)
@ -728,8 +740,10 @@ class TenantFanOutSpanProcessor(SpanProcessor):
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
pending_drains: int = _MAX_PENDING_DRAINS,
drain_pool: _DrainPool | None = None,
langfuse_mapped: bool = True,
) -> None:
self._operator_sinks: Final = operator_sinks
self._langfuse_mapped: Final = langfuse_mapped
self._excluded_db_systems: Final = excluded_db_systems
self._default_capture: Final = default_capture
self._default_capture_by_backend: Final = default_capture_by_backend
@ -822,7 +836,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
try:
processor.on_end(
_scoped(
_for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope
_for_destination(span, destination, self._default_capture_for(destination), self._langfuse_mapped),
destination.span_scope,
)
)
except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span
@ -1198,17 +1213,19 @@ class _OverriddenBackendFilter(SpanProcessor):
return self._inner.force_flush(timeout_millis)
class _LangfuseRequestMetadataFilter(SpanProcessor):
"""Strip the Langfuse request metadata from LiteLLM's own spans before ``inner`` exports them."""
class _DestinationLangfuseAttributeFilter(SpanProcessor):
"""Strip the Langfuse attributes only a Langfuse destination asked for from LiteLLM's own spans before ``inner``
exports them."""
def __init__(self, inner: SpanProcessor) -> None:
def __init__(self, inner: SpanProcessor, langfuse_mapped: bool) -> None:
self._inner: Final = inner
self._langfuse_mapped: Final = langfuse_mapped
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
self._inner.on_start(span, parent_context)
def on_end(self, span: ReadableSpan) -> None:
self._inner.on_end(without_langfuse_request_metadata(span))
self._inner.on_end(without_destination_langfuse_attributes(span, self._langfuse_mapped))
def shutdown(self) -> None:
self._inner.shutdown()
@ -1491,13 +1508,18 @@ def _spec_processor(
spec: ExporterSpec,
use_simple_processor: bool | None,
strip_message_content: bool,
strip_langfuse_request_metadata: bool,
strip_destination_langfuse_attributes: bool,
langfuse_mapped: bool,
) -> SpanProcessor:
exporting: Final = _processor_for(
_exporter_from_spec(spec),
(spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor),
)
scoped: Final = _LangfuseRequestMetadataFilter(exporting) if strip_langfuse_request_metadata else exporting
scoped: Final = (
_DestinationLangfuseAttributeFilter(exporting, langfuse_mapped)
if strip_destination_langfuse_attributes
else exporting
)
return _MessageContentFilter(scoped) if strip_message_content else scoped
@ -1530,6 +1552,7 @@ def build_tracer_provider(
use_simple_processor: bool | None = None,
tenant_overrides: bool = False,
strip_message_content: bool = False,
console_default: bool = False,
) -> TracerProvider:
"""Build the shared :class:`TracerProvider`.
@ -1548,11 +1571,16 @@ def build_tracer_provider(
``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel``
alone; a collector or any other backend in the same config keeps the full tree.
In a config that writes the Langfuse vocabulary, every exporter but ``langfuse_otel``'s
goes without the request metadata the Langfuse mapper writes on LiteLLM's own spans.
Every exporter but ``langfuse_otel``'s goes without the Langfuse attributes only a
Langfuse destination asked for (see :func:`without_destination_langfuse_attributes`),
which any logger writes for a request with a Langfuse destination.
``strip_message_content`` removes prompt and response content before every
exporter, for a routed provider whose spans can carry content another
destination asked for.
The console spec ``config._normalize`` folds in when nothing was configured
prints only under ``console_default``, which the ``otel`` callback passes: a
vendor preset or the proxy's background global logger was never asked to print.
"""
provider: Final = TracerProvider(resource=build_resource(config))
provider.sampler = _RequestExclusionSampler(provider.sampler)
@ -1564,19 +1592,20 @@ def build_tracer_provider(
provider.add_span_processor(_processor_for(exporter, use_simple_processor))
return provider
langfuse_mapped: Final = _maps_langfuse(config)
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
writes_langfuse: Final = "langfuse" in config.mapper_names or any(
spec.owner is ExporterOwner.LANGFUSE_OTEL for spec in config.exporters
)
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
if is_unconfigured_placeholder(spec) and not console_default:
continue
processor = _spec_processor(
spec,
use_simple_processor,
strip_message_content,
strip_langfuse_request_metadata=writes_langfuse and spec.owner is not ExporterOwner.LANGFUSE_OTEL,
strip_destination_langfuse_attributes=spec.owner is not ExporterOwner.LANGFUSE_OTEL,
langfuse_mapped=langfuse_mapped,
)
owner = spec.owner.value if tenant_overrides and spec.owner is not None else None
scope = _operator_scope(config, spec)
@ -1613,8 +1642,9 @@ def attach_tenant_fan_out(
so exactly one fan-out lands. ``configs`` name the operator's own exporters, one
config per v2 logger since each keeps its own provider and still writes its
account, so an additive destination pointing at any of them is delivered once
rather than twice. ``excluded_db_systems`` only filters what the fan-out
delivers, never the operator's own exporters.
rather than twice. The first is the config of the logger that owns ``provider``,
whose model calls the fan-out carries. ``excluded_db_systems`` only filters what
the fan-out delivers, never the operator's own exporters.
"""
with _FAN_OUT_ATTACH_LOCK:
if any(isinstance(processor, TenantFanOutSpanProcessor) for processor in _attached_processors(provider)):
@ -1625,10 +1655,16 @@ def attach_tenant_fan_out(
excluded_db_systems=excluded_db_systems,
default_capture=_operator_capture_mode(configs),
default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))),
langfuse_mapped=any(_maps_langfuse(config) for config in configs[:1]),
)
)
def _maps_langfuse(config: OpenTelemetryV2Config) -> bool:
"""Whether a logger built from ``config`` writes the Langfuse vocabulary on every model call."""
return "langfuse" in config.mapper_names
def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str:
"""The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default"""
return next(

View file

@ -163,10 +163,12 @@ class TenantTracerCache:
config: OpenTelemetryV2Config,
callback_name: str | None,
tracer_name: str,
console_default: bool = False,
) -> None:
self._config = config
self._callback_name = callback_name
self._tracer_name = tracer_name
self._console_default = console_default
# Guards the three mutable structures below: ``pre_call`` can run on
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
@ -284,6 +286,7 @@ class TenantTracerCache:
built: Final = build_tracer_provider(
self._routed_config(credential_headers, project_headers, endpoint, service_name),
strip_message_content=not self._config.capture_span_content,
console_default=self._console_default,
)
self._providers[cache_key] = built
return built

View file

@ -111,7 +111,7 @@ _REQUIRED_HEADERS_BY_CALLBACK: Final[Mapping[str, frozenset[str]]] = MappingProx
_NO_ATTRS: Final[Mapping[str, str]] = MappingProxyType({})
def _arize_project() -> Mapping[str, str]:
def _arize_project(_params: StandardCallbackDynamicParams) -> Mapping[str, str]:
from litellm.integrations.arize.arize import ArizeLogger
project: Final = ArizeLogger.get_arize_config().project_name
@ -124,10 +124,27 @@ def _arize_default_project() -> Mapping[str, str]:
return MappingProxyType({"model_id": ARIZE_DEFAULT_PROJECT})
#: Resource attributes the operator chose for a backend, applied to every export the
#: team receives over the span's own resource: ``ARIZE_PROJECT_NAME`` for Arize.
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str]]]] = MappingProxyType(
{"arize": _arize_project}
def _stands_in_for_the_legacy_langfuse_logger() -> bool:
"""Whether a Langfuse destination replaces the older Langfuse logger's copy, which a proxy without Langfuse keys
of its own sends for a key's or team's ``langfuse_otel``."""
return not (os.environ.get("LANGFUSE_PUBLIC_KEY") and os.environ.get("LANGFUSE_SECRET_KEY"))
def _langfuse_environment(params: StandardCallbackDynamicParams) -> Mapping[str, str]:
"""The key's or team's ``langfuse_environment``, else ``LANGFUSE_TRACING_ENVIRONMENT``, where the older Langfuse
logger wrote it."""
from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
if not _stands_in_for_the_legacy_langfuse_logger():
return _NO_ATTRS
environment: Final = params.get("langfuse_environment") or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
return MappingProxyType({LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value: environment} if environment else {})
#: Resource attributes a backend gets on every export the team receives, over the
#: span's own resource: ``ARIZE_PROJECT_NAME`` for Arize, the Langfuse environment.
_RESOURCE_ATTRIBUTES_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], Mapping[str, str]]]] = (
MappingProxyType({"arize": _arize_project, "langfuse_otel": _langfuse_environment})
)
#: Resource attributes a backend needs on every export to accept it, filled only where
@ -137,12 +154,14 @@ _RESOURCE_DEFAULTS_BY_CALLBACK: Final[Mapping[str, Callable[[], Mapping[str, str
)
def _resource_attributes(callback_name: str, service_name: str | None) -> Mapping[str, str]:
def _resource_attributes(
callback_name: str, service_name: str | None, params: StandardCallbackDynamicParams
) -> Mapping[str, str]:
backend: Final = _RESOURCE_ATTRIBUTES_BY_CALLBACK.get(callback_name)
return MappingProxyType(
{
**({"service.name": service_name} if service_name else {}),
**(backend() if backend is not None else {}),
**(backend(params) if backend is not None else {}),
}
)
@ -214,7 +233,7 @@ def destination_for(
return OtelDestination(
endpoint=endpoint,
headers=MappingProxyType(dict(headers)),
resource_attributes=_resource_attributes(callback_name, service_name),
resource_attributes=_resource_attributes(callback_name, service_name, params),
resource_defaults=_resource_defaults(callback_name),
callback_name=callback_name,
protocol=protocol,

View file

@ -3,7 +3,7 @@
from collections.abc import Iterable
from typing import Final
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec
from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, is_unconfigured_placeholder
def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]:
@ -35,14 +35,3 @@ def credential_gated_exporters(
*(spec for spec in exporters if not is_unconfigured_placeholder(spec)),
ExporterSpec(owner=owner, requires_headers=True),
)
def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool:
"""Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured.
No field set is what says the operator asked for nothing: an exporter they did
configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default,
and so does the gated spec this module appends, which would otherwise eat itself
when one preset layers onto another.
"""
return not spec.model_fields_set

View file

@ -292,6 +292,10 @@ def get_llm_provider(
elif endpoint == "api.deepinfra.com/v1/openai":
custom_llm_provider = "deepinfra"
dynamic_api_key = get_secret_str("DEEPINFRA_API_KEY")
elif endpoint == "api.clfaigateway.dev/v1":
custom_llm_provider = "clf_ai_gateway" # rebind-ok: endpoint inference, like every branch
clf_env_key = get_secret_str("CLF_AI_GATEWAY_API_KEY")
dynamic_api_key = api_key or clf_env_key
elif endpoint == "api.mistral.ai/v1":
custom_llm_provider = "mistral"
dynamic_api_key = get_secret_str("MISTRAL_API_KEY")
@ -644,6 +648,11 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
) = litellm.DeepInfraConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "clf_ai_gateway":
(
api_base, # rebind-ok: resolved base/key are what this function returns, like every branch
dynamic_api_key, # rebind-ok: see above
) = litellm.ClfAiGatewayConfig().get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "empower":
api_base = api_base or get_secret("EMPOWER_API_BASE") or "https://app.empower.dev/api/v1"
dynamic_api_key = api_key or get_secret_str("EMPOWER_API_KEY")

View file

@ -1,6 +1,7 @@
# What is this?
## Common Utility file for Logging handler
# Logging function -> log the exact model details + what's being sent | Non-Blocking
import asyncio
import copy
import datetime
import functools
@ -96,6 +97,7 @@ from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
is_unbilled_non_inference_call,
)
from litellm.litellm_core_utils.llm_cost_calc.finalization import FinalizedCost, PreparedCost
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
bedrock_guardrail_trace,
cost_breakdown_with_guardrail,
@ -200,6 +202,7 @@ from litellm.types.utils import (
TextCompletionResponse,
TranscriptionResponse,
Usage,
batch_chat_usage,
)
from litellm.types.videos.main import VideoObject
from litellm.utils import get_base_model_from_metadata, print_verbose
@ -813,6 +816,11 @@ class Logging(LiteLLMLoggingBaseClass):
# Initialize cost breakdown field
self.cost_breakdown: CostBreakdown | None = None
self.finalized_cost: FinalizedCost | None = None
self.prepared_cost: PreparedCost | None = None
self.cost_allowance_discount: float | None = None
self.cost_finalization_lock = asyncio.Lock()
self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
self.billed_token_rates: BilledTokenRates | None = None
self._provider_reported_cost: float | None = None
@ -874,6 +882,11 @@ class Logging(LiteLLMLoggingBaseClass):
"current_attempt_api_call_start_time": None,
"litellm_params": litellm_params,
}
new_call.finalized_cost = None
new_call.prepared_cost = None
new_call.cost_allowance_discount = None
new_call.cost_finalization_lock = asyncio.Lock()
new_call.deferred_success_result = None
new_call.streaming_chunks = []
new_call.sync_streaming_chunks = []
return new_call
@ -2079,6 +2092,11 @@ class Logging(LiteLLMLoggingBaseClass):
elif router_model_id is None and "model_id" in hidden_params: # use model_id if not already set
router_model_id = hidden_params["model_id"]
if isinstance(priced_result, LiteLLMBatch) and (
not batch_cost_is_final(priced_result) or self._is_batch_poll_without_logging(priced_result)
):
return 0.0
# Fallback: extract router_model_id from litellm_params when not available
# from the result object. ResponsesAPIResponse objects (used by /v1/responses
# streaming) don't carry _hidden_params["model_id"] like ModelResponse does.
@ -2462,6 +2480,55 @@ class Logging(LiteLLMLoggingBaseClass):
**kwargs,
)
def prepare_response_cost(self, response: object, now: datetime.datetime) -> PreparedCost | None:
if self.prepared_cost is not None:
return self.prepared_cost
recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
deferred: Final = self.deferred_success_result
prepared: Final = self.prepare_success_logging(
result=deferred[0] if deferred is not None else response,
start_time=deferred[1] if deferred is not None else self.start_time,
end_time=deferred[2] if deferred is not None else now.replace(tzinfo=None),
cache_hit=recorded_cache_hit if isinstance(recorded_cache_hit, bool) else None,
standard_logging_object=None,
build_logging_payload=True,
publish_logging_payload=False,
)
payload: Final = prepared[3]
raw_cost: Final[object] = self.model_call_details.get("response_cost")
if payload is None or not isinstance(raw_cost, (int, float)):
return None
recorded_usage: Final[object] = self.model_call_details.get("combined_usage_object")
response_usage: Final[object] = getattr(prepared[2], "usage", None)
normalized_usage: Final = (
recorded_usage
if isinstance(recorded_usage, Usage)
else response_usage
if isinstance(response_usage, Usage)
else None
)
usage: Final = StandardLoggingPayloadSetup.get_usage_from_response_obj(
_extract_response_obj_and_hidden_params(prepared[2], None)[0],
combined_usage_object=normalized_usage,
).model_copy(deep=True)
self.prepared_cost = PreparedCost(
raw_cost,
{
**payload,
"response": None,
"messages": None,
"model_map_information": copy.deepcopy(payload["model_map_information"]),
"cost_breakdown": copy.deepcopy(payload["cost_breakdown"]),
},
usage,
)
self.record_partial_usage_for_failure(usage, raw_cost)
return self.prepared_cost
def record_finalized_cost(self, finalized: FinalizedCost) -> None:
self.finalized_cost = finalized
self.model_call_details["response_cost"] = finalized.amount
def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
"""Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
self.model_call_details["combined_usage_object"] = usage
@ -2812,12 +2879,16 @@ class Logging(LiteLLMLoggingBaseClass):
logging_result, "_hidden_params", {}
)
if self.model_call_details.get("cache_hit") is True:
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("cache_hit") is True:
self.model_call_details["response_cost"] = 0.0
elif "response_cost" in hidden_params:
self.model_call_details["response_cost"] = hidden_params["response_cost"]
self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"])
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and existing_cost != 0:
elif (existing_cost := self.model_call_details.get("response_cost")) is not None and (
existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped
):
# Preserve response_cost if already calculated (e.g., by pass-through
# handlers like Gemini/Vertex which call completion_cost directly).
# Do not preserve 0 from failure_handler on intermediate router retries.
@ -2936,7 +3007,7 @@ class Logging(LiteLLMLoggingBaseClass):
standard_logging_object: StandardLoggingPayload | None = None,
build_logging_payload: bool = True,
) -> tuple[datetime.datetime, datetime.datetime, object]:
prepared: Final = self._prepare_success_logging(
prepared: Final = self.prepare_success_logging(
result=result,
start_time=start_time,
end_time=end_time,
@ -2947,7 +3018,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
return prepared[0], prepared[1], prepared[2]
def _prepare_success_logging(
def prepare_success_logging(
self,
result: object,
start_time: datetime.datetime | None,
@ -2968,7 +3039,11 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
self.model_call_details["cache_hit"] = cache_hit
self.model_call_details["cache_hit"] = (
self.prepared_cost.payload["cache_hit"]
if cache_hit is None and self.prepared_cost is not None
else cache_hit
)
if self.call_type == CallTypes.anthropic_messages.value:
result = self._anthropic_messages_logged_response(result=result)
@ -3007,6 +3082,7 @@ class Logging(LiteLLMLoggingBaseClass):
if (
litellm.max_budget
and publish_logging_payload
and self.stream is False
and result is not None
and isinstance(result, dict)
@ -3037,10 +3113,17 @@ class Logging(LiteLLMLoggingBaseClass):
publish_logging_payload: bool,
) -> StandardLoggingPayload | None:
if standard_logging_object is not None:
effective_payload: Final = (
self.finalized_cost.logging_payload(standard_logging_object)
if self.finalized_cost is not None
else standard_logging_object
)
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
if not publish_logging_payload:
return standard_logging_object.copy()
self.model_call_details["standard_logging_object"] = standard_logging_object
return standard_logging_object
return effective_payload.copy()
self.model_call_details["standard_logging_object"] = effective_payload
return effective_payload
if result is None or self.stream is True:
# Streaming reaches here before its cost is known, so the cost
# is seeded to None, but only when nothing has already
@ -3269,6 +3352,10 @@ class Logging(LiteLLMLoggingBaseClass):
finally:
self._restore_correlation_context()
def _is_batch_poll_without_logging(self, result: object) -> bool:
litellm_metadata: Final = (self.litellm_params or {}).get("litellm_metadata") or {}
return isinstance(result, LiteLLMBatch) and litellm_metadata.get("batch_ignore_default_logging", False) is True
def _success_handler_body(
self,
result: object = None,
@ -3281,7 +3368,7 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Success Call: Cache_hit=%s", cache_hit)
if not self.should_run_logging(event_type="sync_success"): # prevent double logging
return
start_time, end_time, result, request_logging_payload = self._prepare_success_logging(
start_time, end_time, result, request_logging_payload = self.prepare_success_logging(
start_time=start_time,
end_time=end_time,
result=result,
@ -3783,11 +3870,7 @@ class Logging(LiteLLMLoggingBaseClass):
## CALCULATE COST FOR BATCH JOBS
if self.call_type == CallTypes.aretrieve_batch.value and isinstance(result, LiteLLMBatch):
litellm_params = self.litellm_params or {}
litellm_metadata: Final = litellm_params.get("litellm_metadata") or {}
if (
litellm_metadata.get("batch_ignore_default_logging", False) is True
): # polling job will query these frequently, don't spam db logs
if self._is_batch_poll_without_logging(result):
return
batch_cost: Final = kwargs.get("batch_cost", None)
@ -4204,7 +4287,9 @@ class Logging(LiteLLMLoggingBaseClass):
# chunks already delivered; the router stashes that recovered usage as
# ``combined_usage_object`` and pre-computes its cost, so preserve it
# here instead of zeroing the spend on an otherwise-failed request.
if self.model_call_details.get("combined_usage_object") is None:
if self.finalized_cost is not None:
self.model_call_details["response_cost"] = self.finalized_cost.amount
elif self.model_call_details.get("combined_usage_object") is None:
self.model_call_details["response_cost"] = 0
if hasattr(exception, "headers") and isinstance(exception.headers, dict):
@ -5544,7 +5629,7 @@ def build_custom_logger_compatible_class(
return callback
otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration)
otel_logger_v2: Final = build_otel_v2_logger(
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings
config=OpenTelemetryV2Config(**otel_settings), settings=otel_settings, console_default=True
)
_in_memory_loggers.append(otel_logger_v2)
_maybe_auto_initialize_arize_phoenix(_in_memory_loggers)
@ -7367,6 +7452,12 @@ def _cache_hit_cost_breakdown(served: CostBreakdown | None) -> CostBreakdown:
)
def _finalized_cost_of(logging_obj: object) -> FinalizedCost | None:
"""Duck-typed logging objects (and test doubles) carry no finalized cost."""
finalized: Final = getattr(logging_obj, "finalized_cost", None)
return finalized if isinstance(finalized, FinalizedCost) else None
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: object,
@ -7396,7 +7487,8 @@ def get_standard_logging_object_payload(
# Extract usage as a plain dict, avoiding Pydantic round-trip
raw_usage_dict: Final = StandardLoggingPayloadSetup.get_usage_as_dict(
response_obj=None if is_unbilled_non_inference_call(call_type, metadata, response_obj) else response_obj,
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object")),
combined_usage_object=cast(Usage | None, kwargs.get("combined_usage_object"))
or (batch_chat_usage(init_response_obj) if isinstance(init_response_obj, LiteLLMBatch) else None),
)
usage_dict: Final = (
{**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
@ -7470,7 +7562,10 @@ def get_standard_logging_object_payload(
raw_response_cost: Final = kwargs.get("response_cost")
llm_response_cost: Final[float] = raw_response_cost or 0.0
guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information"))
response_cost: Final[float] = llm_response_cost + guardrail_cost
finalized_cost: Final = _finalized_cost_of(logging_obj)
response_cost: Final[float] = (
finalized_cost.amount if finalized_cost is not None else llm_response_cost + guardrail_cost
)
# clean up litellm hidden params
clean_hidden_params: Final = StandardLoggingPayloadSetup.get_hidden_params(hidden_params)
@ -7537,13 +7632,17 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
request_cost_breakdown: Final = (
finalized_cost.logging_breakdown()
if finalized_cost is not None
else cost_breakdown_with_guardrail(
(
_cache_hit_cost_breakdown(uncached_price.cost_breakdown)
if uncached_price is not None
else logging_obj.cost_breakdown
),
guardrail_cost,
)
)
captured_baseline: Final = logging_obj.baseline_observation
autorouter_savings: Final = (
@ -7600,6 +7699,7 @@ def get_standard_logging_object_payload(
model=model_name,
metadata=clean_metadata,
cache_key=clean_hidden_params["cache_key"],
cost_finalized=finalized_cost is not None,
response_cost=response_cost,
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,

View file

@ -0,0 +1,61 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final, TypeAlias
from pydantic import TypeAdapter
from litellm.types.utils import CostBreakdown, StandardLoggingPayload, Usage
BreakdownValue: TypeAlias = float | str | None | Mapping[str, float]
_BREAKDOWN_FIELDS: Final = TypeAdapter(Mapping[str, BreakdownValue])
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
@dataclass(frozen=True, slots=True)
class PreparedCost:
amount: float
payload: StandardLoggingPayload
usage: Usage
@dataclass(frozen=True, slots=True)
class FinalizedCost:
amount: float
breakdown: Mapping[str, BreakdownValue] | None
allowance_discount: float
def logging_payload(self, payload: StandardLoggingPayload) -> StandardLoggingPayload:
return {
**payload,
"response_cost": self.amount,
"cost_breakdown": self.logging_breakdown(),
"cost_finalized": True,
}
def logging_breakdown(self) -> CostBreakdown | None:
return _COST_BREAKDOWN.validate_python(self.breakdown) if self.breakdown is not None else None
def finalize_cost(amount: float, breakdown: CostBreakdown | None, allowance_discount: float = 0.0) -> FinalizedCost:
discount: Final = min(max(amount, 0.0), max(allowance_discount, 0.0))
total: Final = amount - discount
existing: Final = _BREAKDOWN_FIELDS.validate_python(breakdown) if breakdown is not None else None
adjusted: Final = (
{
**(existing or {}),
"total_cost": total,
"original_cost": (breakdown or {}).get("original_cost", amount),
"discount_amount": (breakdown or {}).get("discount_amount", 0.0) + discount,
}
if discount > 0
else existing
)
frozen: Final = (
MappingProxyType(
{key: MappingProxyType(value) if isinstance(value, Mapping) else value for key, value in adjusted.items()}
)
if adjusted is not None
else None
)
return FinalizedCost(amount=total, breakdown=frozen, allowance_discount=discount)

View file

@ -75,6 +75,14 @@ _IMAGE_TOKEN_RATE_KEYS: Final[tuple[str, ...]] = (
"output_cost_per_image_token",
)
def _deployment_prices_images_without_an_output_price(pricing: ModelInfo | None) -> bool:
return pricing is not None and (
"input_cost_per_pixel" in pricing
or ("input_cost_per_image" in pricing and "output_cost_per_image" not in pricing)
)
# Pre-resolved service-tier cost-key suffixes (e.g. "_priority"). Used per
# request in the cost-calc path, so the f-strings are built once here instead
# of being rebuilt for every model_info key on every call. Longest-first so a
@ -127,7 +135,7 @@ def _get_token_detail_value(details: object, key: str) -> int | None:
return value if isinstance(value, int) else None
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-)\d+")
_IMAGE_SIZE_PATTERN: Final = re.compile(r"\d+(?:x|-x-|\*)\d+")
def _requested_image_param(optional_params: Mapping[str, object] | None, key: str) -> str | None:
@ -137,7 +145,7 @@ def _requested_image_param(optional_params: Mapping[str, object] | None, key: st
def _requested_image_size(optional_params: Mapping[str, object] | None) -> str | None:
value: Final = _requested_image_param(optional_params, "size")
return value if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
return value.replace("*", "x") if value is not None and _IMAGE_SIZE_PATTERN.fullmatch(value) else None
def get_web_search_requests(server_tool_use: object) -> int | None:
@ -2090,7 +2098,7 @@ class _ModelPricedImageCostCalculator(Protocol):
def _model_priced_image_cost_calculator(
custom_llm_provider: str | None, model: str
custom_llm_provider: str | None, model: str, pricing: ModelInfo | None
) -> _ModelPricedImageCostCalculator | None:
match custom_llm_provider:
case litellm.LlmProviders.RECRAFT.value:
@ -2116,6 +2124,14 @@ def _model_priced_image_cost_calculator(
from litellm.llms.tencent.image_generation.transformation import is_tokenhub_image_model
return tencent_image_cost_calculator if is_tokenhub_image_model(model) else None
case litellm.LlmProviders.DASHSCOPE.value:
if _deployment_prices_images_without_an_output_price(pricing):
return None
from litellm.llms.dashscope.image_generation.cost_calculator import (
cost_calculator as dashscope_image_cost_calculator,
)
return dashscope_image_cost_calculator
case _:
return None
@ -2178,7 +2194,7 @@ class CostCalculatorUtils:
)
resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
pricing: Final = deployment_pricing(model_info)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model)
model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
if model_priced_calculator is not None:
return model_priced_calculator(model=model, image_response=completion_response, model_info=pricing)

View file

@ -0,0 +1,45 @@
"""How strong each reasoning_effort level is, and rounding a requested level onto a declared set.
Imports nothing from the SDK, so the `lite` CLI, which loads without initializing litellm, can use it.
"""
from collections.abc import Sequence
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing_extensions import assert_never
ReasoningEffortRounding: TypeAlias = Literal["up", "down"]
REASONING_EFFORT_STRENGTH_ORDER: Final = ("minimal", "low", "medium", "high", "xhigh", "max")
REASONING_EFFORT_STRENGTH_RANK: Final = MappingProxyType(
{effort: rank for rank, effort in enumerate(REASONING_EFFORT_STRENGTH_ORDER)}
)
def nearest_declared_reasoning_effort(
requested: str, declared: Sequence[str], rounding: ReasoningEffortRounding = "up"
) -> str:
"""Rounding up takes the weakest declared level at least as strong as the request, else the
strongest declared, so the caller gets no less reasoning than it asked for instead of a rejected
call. Rounding down mirrors it: the strongest declared level at most as strong, else the
weakest. none is the off switch rather than a strength, so it is never rounded onto the ladder
and no level is rounded to it: a caller who turned reasoning off must not be billed for it, and
a model that cannot turn it off says so itself. A level outside the strength order is likewise
returned as is for upstream to judge."""
ranked: Final = sorted(
(effort for effort in declared if effort in REASONING_EFFORT_STRENGTH_RANK),
key=lambda effort: REASONING_EFFORT_STRENGTH_RANK[effort],
)
if requested in ranked or requested not in REASONING_EFFORT_STRENGTH_RANK or not ranked:
return requested
strength: Final = REASONING_EFFORT_STRENGTH_RANK[requested]
match rounding:
case "up":
return next((effort for effort in ranked if REASONING_EFFORT_STRENGTH_RANK[effort] >= strength), ranked[-1])
case "down":
return next(
(effort for effort in reversed(ranked) if REASONING_EFFORT_STRENGTH_RANK[effort] <= strength), ranked[0]
)
case _:
assert_never(rounding)

View file

@ -0,0 +1,27 @@
from collections.abc import Iterable
from typing import Final
from litellm.types.utils import ModelResponseStream, StreamingChoices
def is_empty_streaming_choice(choice: StreamingChoices) -> bool:
if choice.finish_reason is not None:
return False
if getattr(choice, "logprobs", None) is not None:
return False
delta: Final = choice.delta if hasattr(choice, "delta") else None
if delta is None:
return True
return _all_values_none(delta.model_dump().values())
def _all_values_none(values: Iterable[object]) -> bool:
return all(value is None for value in values)
def is_injected_stream_usage_artifact(chunk: object) -> bool:
if not isinstance(chunk, ModelResponseStream):
return False
if chunk.provider_specific_fields is not None:
return False
return all(is_empty_streaming_choice(choice) for choice in chunk.choices or [])

View file

@ -3,7 +3,7 @@ import builtins
import time
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass, replace
from itertools import chain, groupby
from itertools import chain, groupby, takewhile
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, Protocol, TypeAlias, TypedDict, TypeVar, Union, cast
@ -257,7 +257,7 @@ def apply_grounding_request_counts(
class ChunkProcessor:
def __init__(self, chunks: list, messages: Sequence | None = None):
def __init__(self, chunks: list, messages: Sequence[object] | None = None):
self.chunks = self._sort_chunks(chunks)
self.messages = messages
self.first_chunk = chunks[0]
@ -390,6 +390,30 @@ class ChunkProcessor:
case _:
return "assistant"
@staticmethod
def _finish_reason_of_chunks(chunks: Sequence["_BaseChunk"], unreported: str = "stop") -> object:
finish_reason = unreported
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
return finish_reason
@staticmethod
def build_base_choice(index: int, chunks: Sequence["_BaseChunk"]) -> Choices:
return Choices(
index=index,
finish_reason=ChunkProcessor._finish_reason_of_chunks(
chunks, unreported="tool_calls" if _carries_tool_calls(chunks) else "stop"
),
message={"role": ChunkProcessor._get_role_from_chunks(chunks), "content": ""},
)
@staticmethod
def _get_model_from_chunks(chunks: Sequence["_BaseChunk"], first_chunk_model: str) -> str:
"""
@ -414,7 +438,11 @@ class ChunkProcessor:
None,
)
def build_base_response(self, chunks: Sequence["_BaseChunk"]) -> ModelResponse:
def build_base_response(
self,
chunks: Sequence["_BaseChunk"],
choice_streams: Sequence[tuple[int, Sequence["_BaseChunk"]]],
) -> ModelResponse:
chunk = self.first_chunk
id: Final = ChunkProcessor._get_chunk_id(chunks)
object: Final = chunk["object"]
@ -426,16 +454,7 @@ class ChunkProcessor:
service_tier: Final = ChunkProcessor._get_service_tier_from_chunks(chunks)
role: Final = ChunkProcessor._get_role_from_chunks(chunks)
finish_reason = "stop"
for chunk in chunks:
if "choices" in chunk and len(chunk["choices"]) > 0:
chunk_finish_reason = None
if hasattr(chunk["choices"][0], "finish_reason"):
chunk_finish_reason = chunk["choices"][0].finish_reason
elif "finish_reason" in chunk["choices"][0]:
chunk_finish_reason = _CHOICE_FIELD_VALUE.validate_python(chunk["choices"][0]["finish_reason"])
if chunk_finish_reason is not None:
finish_reason = chunk_finish_reason
finish_reason: Final = ChunkProcessor._finish_reason_of_chunks(chunks)
# Initialize the response dictionary
response = ModelResponse(
@ -455,13 +474,17 @@ class ChunkProcessor:
if service_tier is not None
else MappingProxyType({})
),
"choices": [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
],
"choices": (
[ChunkProcessor.build_base_choice(index, choice_chunks) for index, choice_chunks in choice_streams]
if len(choice_streams) > 1
else [
{
"index": 0,
"message": {"role": role, "content": ""},
"finish_reason": finish_reason,
}
]
),
"usage": {
"prompt_tokens": 0, # Modify as needed
"completion_tokens": 0, # Modify as needed
@ -1520,11 +1543,188 @@ def _bedrock_trace_fields(chunk: object) -> Mapping[str, object]:
return {field: fields[field] for field in BEDROCK_TRACE_FIELDS if field in fields}
def _stream_choice_index(choice: object) -> int:
index: Final = _field_of(choice, "index")
return index if isinstance(index, int) else 0
def _carries_tool_calls(chunks: Sequence[_BaseChunk]) -> bool:
return any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"]
for choice in chain.from_iterable(chunk.get("choices") or () for chunk in chunks)
)
def _choice_streams(chunks: Sequence[_BaseChunk]) -> tuple[tuple[int, Sequence[_BaseChunk]], ...]:
indices: Final = sorted(
frozenset(map(_stream_choice_index, chain.from_iterable(chunk.get("choices") or () for chunk in chunks)))
)
if len(indices) <= 1:
return ((0, chunks),)
return tuple((index, _narrowed_to_choice(chunks, index)) for index in indices)
def _narrowed_to_choice(chunks: Sequence[_BaseChunk], index: int) -> tuple[_BaseChunk, ...]:
return tuple(
_BaseChunk(choices=[choice for choice in chunk.get("choices") or () if _stream_choice_index(choice) == index])
for chunk in chunks
)
def _simple_text(chunks: Sequence[_BaseChunk]) -> str | None:
choices_per_chunk: Final = tuple(chunk["choices"] for chunk in chunks if chunk.get("choices"))
parts: Final = tuple(takewhile(lambda part: part is not None, map(_simple_text_part, choices_per_chunk)))
return "".join(filter(None, parts)) if len(parts) == len(choices_per_chunk) else None
def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> Iterator[tuple[str, object]]:
tool_call_chunks: Final = [
chunk
for chunk in chunks
if any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
for choice in chunk.get("choices") or ()
)
]
if tool_call_chunks:
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
yield "content", None
yield "tool_calls", tool_calls_list
function_call_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "function_call" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["function_call"] is not None
]
if function_call_chunks:
yield "content", None
yield "function_call", processor.get_combined_function_call_content(function_call_chunks)
content_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["content"] is not None
]
if content_chunks:
yield "content", processor.get_combined_content(content_chunks)
thinking_blocks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "thinking_blocks" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
]
if thinking_blocks:
yield "thinking_blocks", processor.get_combined_thinking_content(thinking_blocks)
reasoning_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "reasoning_content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
]
if reasoning_chunks:
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
yield "reasoning_items", list(reasoning_items)
annotation_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "annotations" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["annotations"] is not None
]
message_annotations, provider_annotations = (
partition_annotations(
list(chain.from_iterable(chunk["choices"][0]["delta"]["annotations"] for chunk in annotation_chunks))
)
if annotation_chunks
else (None, ())
)
if message_annotations is not None:
yield "annotations", message_annotations
audio_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "audio" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["audio"] is not None
]
if audio_chunks:
yield "audio", processor.get_combined_audio_content(audio_chunks)
# Handle image chunks from models like gemini-2.5-flash-image
# See: https://github.com/BerriAI/litellm/issues/19478
image_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "images" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["images"] is not None
]
if image_chunks:
# Images come complete in a single chunk, collect all images from all chunks
all_images: Final = []
for chunk in image_chunks:
all_images.extend(chunk["choices"][0]["delta"]["images"])
yield "images", all_images
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
# See: https://github.com/BerriAI/litellm/issues/17737
provider_specific_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "provider_specific_fields" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
]
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = {"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
provider_annotation_fields: Final = {"annotations": list(provider_annotations)} if provider_annotations else {}
combined_provider_fields: Final = {
key: value
for fields in (provider_annotation_fields, citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
yield "provider_specific_fields", combined_provider_fields
def stream_chunk_builder(
chunks: list,
messages: Sequence | None = None,
start_time=None,
end_time=None,
start_time: object = None,
end_time: object = None,
logging_obj: Optional["Logging"] = None,
count_prompt_tokens: Callable[[], int] | None = None,
) -> ModelResponse | TextCompletionResponse | None:
@ -1561,26 +1761,15 @@ def stream_chunk_builder(
usage_model: Final = usage_tokenizer_model(
model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
)
# Initialize the response dictionary
response: Final = processor.build_base_response(chunks)
choice_streams: Final = _choice_streams(chunks)
response: Final = processor.build_base_response(chunks, choice_streams)
rebuilt_choices: Final = tuple(zip(response.choices, (choice_chunks for _, choice_chunks in choice_streams)))
# Fast path for the common text-only streaming case:
# avoid repeated multi-pass list scans over chunks.
simple_content_parts: Final[list[str]] = []
is_simple_text_stream = True
for chunk in chunks:
if not chunk.get("choices"):
continue
if (part := _simple_text_part(chunk["choices"])) is None:
is_simple_text_stream = False
break
if part:
simple_content_parts.append(part)
if is_simple_text_stream:
if simple_content_parts:
response["choices"][0]["message"]["content"] = "".join(simple_content_parts)
simple_texts: Final = tuple(_simple_text(choice_chunks) for _, choice_chunks in choice_streams)
if None not in simple_texts:
for (choice, _), simple_text in zip(rebuilt_choices, simple_texts):
if simple_text:
choice.message.content = simple_text
completion_output = get_content_from_model_response(response)
usage = processor.calculate_usage(
chunks=chunks,
@ -1619,165 +1808,9 @@ def stream_chunk_builder(
processor.apply_provider_assembled_streaming_metadata(response, chunks, logging_obj)
return response
tool_call_chunks: Final = [
chunk
for chunk in chunks
if any(
"tool_calls" in choice["delta"] and choice["delta"]["tool_calls"] is not None
for choice in chunk.get("choices") or ()
)
]
if len(tool_call_chunks) > 0:
tool_calls_list: Final = processor.get_combined_tool_content(tool_call_chunks)
_choice = response.choices[0]
_choice.message.content = None
_choice.message.tool_calls = tool_calls_list
function_call_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "function_call" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["function_call"] is not None
]
if len(function_call_chunks) > 0:
_choice = response.choices[0]
_choice.message.content = None
_choice.message.function_call = processor.get_combined_function_call_content(function_call_chunks)
content_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["content"] is not None
]
if len(content_chunks) > 0:
response["choices"][0]["message"]["content"] = processor.get_combined_content(content_chunks)
thinking_blocks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "thinking_blocks" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["thinking_blocks"] is not None
]
if len(thinking_blocks) > 0:
response["choices"][0]["message"]["thinking_blocks"] = processor.get_combined_thinking_content(
thinking_blocks
)
reasoning_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "reasoning_content" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["reasoning_content"] is not None
]
if len(reasoning_chunks) > 0:
response["choices"][0]["message"]["reasoning_content"] = processor.get_combined_reasoning_content(
reasoning_chunks
)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
response["choices"][0]["message"]["reasoning_items"] = list(reasoning_items)
annotation_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "annotations" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["annotations"] is not None
]
if len(annotation_chunks) > 0:
# Merge annotations from ALL chunks — providers may spread
# them across multiple streaming chunks or send them only in
# the final chunk.
all_annotations: Final[list[object]] = []
for ac in annotation_chunks:
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
message_annotations, provider_annotations = partition_annotations(all_annotations)
if message_annotations is not None:
response["choices"][0]["message"]["annotations"] = message_annotations
if provider_annotations:
response.choices[0].message.provider_specific_fields = {
**(response.choices[0].message.provider_specific_fields or {}),
"annotations": list(provider_annotations),
}
audio_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "audio" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["audio"] is not None
]
if len(audio_chunks) > 0:
_choice = response.choices[0]
_choice.message.audio = processor.get_combined_audio_content(audio_chunks)
# Handle image chunks from models like gemini-2.5-flash-image
# See: https://github.com/BerriAI/litellm/issues/19478
image_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "images" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["images"] is not None
]
if len(image_chunks) > 0:
# Images come complete in a single chunk, collect all images from all chunks
all_images: Final = []
for chunk in image_chunks:
all_images.extend(chunk["choices"][0]["delta"]["images"])
response["choices"][0]["message"]["images"] = all_images
# Combine provider_specific_fields from streaming chunks (e.g., web_search_results, citations)
# See: https://github.com/BerriAI/litellm/issues/17737
provider_specific_chunks: Final = [
chunk
for chunk in chunks
if chunk.get("choices")
and "provider_specific_fields" in chunk["choices"][0]["delta"]
and chunk["choices"][0]["delta"]["provider_specific_fields"] is not None
]
if len(provider_specific_chunks) > 0:
provider_field_dicts: Final = tuple(
fields
for chunk in provider_specific_chunks
for fields in (chunk["choices"][0]["delta"]["provider_specific_fields"],)
if isinstance(fields, dict)
)
streamed_citations: Final = tuple(
fields["citation"] for fields in provider_field_dicts if fields.get("citation") is not None
)
citation_fields: Final = (
{"citations": _joined_streamed_citations(streamed_citations)} if streamed_citations else {}
)
combined_provider_fields: Final = {
key: value
for fields in (citation_fields, *provider_field_dicts)
for key, value in fields.items()
if key != "citation"
}
if combined_provider_fields:
_choice = response.choices[0]
_choice.message.provider_specific_fields = {
**(_choice.message.provider_specific_fields or {}),
**combined_provider_fields,
}
for choice, choice_chunks in rebuilt_choices:
for field, value in _assembled_message_fields(processor, choice_chunks):
choice.message[field] = value
completion_output = get_content_from_model_response(response)

View file

@ -7,9 +7,11 @@ import re
import struct
from bisect import bisect_right
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from itertools import accumulate
from dataclasses import dataclass
from functools import cached_property
from itertools import accumulate, chain, islice
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal, cast
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, cast
import anyio
import anyio.lowlevel
@ -33,7 +35,12 @@ from litellm.constants import (
MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES,
MAX_TILE_HEIGHT,
MAX_TILE_WIDTH,
OPENAI_CHAT_MODES,
OPENAI_MODEL_NAME_MAX_DASHES,
OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS,
PDF_DATA_URL_PREFIX,
PDF_MAX_FORM_XOBJECT_DEPTH,
PDF_MAX_XOBJECT_DRAWS_PER_PAGE,
TIKTOKEN_ENCODE_CHUNK_SIZE_CHARS,
TOKEN_COUNTER_MAX_CONCURRENT_COUNTS,
TOKEN_COUNTER_MAX_EXACT_CHARS,
@ -606,6 +613,35 @@ TokenCounterFunction = Callable[[str], int]
Type for a function that counts tokens in a string.
"""
PdfPagePricing: TypeAlias = Literal["openai", "claude"]
class _PdfBox(Protocol):
@property
def width(self) -> float: ...
@property
def height(self) -> float: ...
class _PdfContent(Protocol):
def get_data(self) -> bytes: ...
class _PdfPage(Protocol):
@property
def mediabox(self) -> _PdfBox: ...
def extract_text(self) -> str: ...
def get_contents(self) -> _PdfContent | None: ...
class _PdfReader(Protocol):
@property
def pages(self) -> Sequence[_PdfPage]: ...
EXTRAPOLATION_SAMPLES: Final = 16
T_ParamSpec: Final = ParamSpec("T_ParamSpec")
T_Retval = TypeVar("T_Retval")
@ -662,7 +698,7 @@ class _MessageCountParams:
if uses_legacy_message_accounting(model):
self.tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
self.tokens_per_name = -1 # if there's a name, the role is omitted
elif actual_model in litellm.open_ai_chat_completion_models or actual_model in litellm.azure_llms:
elif _is_openai_chat_model(actual_model):
self.tokens_per_message = 3
self.tokens_per_name = 1
else:
@ -671,6 +707,63 @@ class _MessageCountParams:
self.tokens_per_name = 1
self.count_function = _get_count_function(model, custom_tokenizer)
self.tool_section = chat_template_tool_section(model)
self._model = model
@cached_property
def pdf_pricing(self) -> PdfPagePricing:
return pdf_page_pricing(self._model)
def _is_openai_chat_model(model: str) -> bool:
return model in litellm.open_ai_chat_completion_models or model in litellm.azure_llms
def _openai_deployment_name(model: str) -> str:
name: Final = model.rpartition("/")[2]
return name.split(":")[1] if name.startswith("ft:") else name
def _openai_model_family(model: str) -> str | None:
words: Final = _openai_deployment_name(model).split("-", OPENAI_MODEL_NAME_MAX_DASHES)
prefixes: Final = ("-".join(words[:end]) for end in range(len(words), 0, -1))
listed: Final = next((prefix for prefix in prefixes if _is_openai_chat_model(prefix)), None)
return None if listed is None or _is_listed_non_chat_model(listed) else listed
def _untyped_cost_map() -> object:
return litellm.model_cost # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # loaded untyped
def _cost_map_entry(model: str) -> Mapping[str, object]:
cost_map: Final = _pdf_dictionary(_untyped_cost_map()) or {}
return _pdf_dictionary(cost_map.get(model)) or {}
def _is_azure_openai_chat_model(model: str) -> bool:
name: Final = _openai_deployment_name(model)
entry: Final = _cost_map_entry(f"azure/{name}")
return (
entry.get("litellm_provider") == "azure"
and entry.get("mode") in OPENAI_CHAT_MODES
and (entry.get("supports_pdf_input") is True or name.startswith("gpt-"))
)
def _is_openai_or_azure_model(model: str) -> bool:
if _is_listed_non_chat_model(_openai_deployment_name(model)):
return False
return _is_openai_chat_model(model) or _openai_model_family(model) is not None or _is_azure_openai_chat_model(model)
def pdf_page_pricing(model: str | None) -> PdfPagePricing:
return "openai" if _is_openai_or_azure_model(model or "") else "claude"
def reads_pdf_pages(model: str) -> bool:
segments: Final = model.split("/")
names: Final = ("/".join(segments[start:]) for start in range(len(segments)))
stated: Final = (_cost_map_entry(name).get("supports_pdf_input") for name in names)
return next((support for support in stated if support is not None), False) is True
def token_counter(
@ -842,6 +935,7 @@ def _count_messages(
value,
use_default_image_token_count,
default_token_count,
params.pdf_pricing,
)
elif key == "search_results" and isinstance(value, list):
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -986,7 +1080,7 @@ def _get_exact_count_function(
return count_tokens
if tokenizer_json["type"] == "openai_tokenizer":
encoding: Final = openai_tokenizer_encoding(model)
encoding: Final = openai_tokenizer_encoding(model or "")
def encode_length(text: str) -> int:
return _encoding_count(encoding, text)
@ -1006,6 +1100,11 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
return get_encoding(openai_tokenizer_encoding_name(model))
def _is_listed_non_chat_model(model: str) -> bool:
modes: Final = (_cost_map_entry(name).get("mode") for name in (model, f"azure/{model}"))
return any(mode is not None and mode not in OPENAI_CHAT_MODES for mode in modes)
def openai_tokenizer_encoding_name(model: str) -> str:
"""The tiktoken encoding name for `model`, without loading the encoding.
@ -1021,7 +1120,9 @@ def openai_tokenizer_encoding_name(model: str) -> str:
try:
return tiktoken.encoding_name_for_model(model_to_use)
except KeyError:
return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
return _suffixed_variant_encoding_name(model_to_use) or (
"cl100k_base" if _is_listed_non_chat_model(model_to_use) else "o200k_base"
)
def unresolved_tokenizer_model(model: str) -> str:
@ -1079,7 +1180,16 @@ def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
def _openai_model_name(model: str) -> str | None:
return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
return (
_listed_openai_model_name(model)
or _tiktoken_o200k_model_name(model)
or _versioned_openai_model_name(model)
or _azure_openai_chat_model_name(model)
)
def _azure_openai_chat_model_name(model: str) -> str | None:
return _openai_deployment_name(model) if _is_azure_openai_chat_model(model) else None
_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
@ -1224,10 +1334,6 @@ def _validate_anthropic_content(content: Mapping[str, object]) -> type:
if expected_cls is None:
raise ValueError(f"Unknown Anthropic content type: '{content_type}'")
missing: Final = [k for k in getattr(expected_cls, "__required_keys__", set()) if k not in content]
if missing:
raise ValueError(f"Missing required fields in {content_type} block: {', '.join(missing)}")
return expected_cls
@ -1253,7 +1359,165 @@ def _anthropic_rendered_page_image_tokens(width: float, height: float) -> int:
return math.ceil(min(float(ANTHROPIC_IMAGE_MAX_PIXELS), area_at_max_edge) / ANTHROPIC_IMAGE_PIXELS_PER_TOKEN)
def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction) -> int | None:
def _pdf_dictionary(node: object) -> Mapping[str, object] | None:
return node if isinstance(node, Mapping) else None # pyright: ignore[reportUnknownVariableType] # read per key
def _pdf_value(container: Mapping[str, object], key: str) -> object:
return container[key] if key in container else None # noqa: SIM401 # pypdf resolves references only in []
_PdfXObjects: TypeAlias = Mapping[str, object] | None
_PdfXObjectUse: TypeAlias = tuple[Mapping[str, object], _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfContentSummary:
drawn_names: tuple[str, ...]
draws_inline_image: bool
_PdfDrawing: TypeAlias = tuple[_PdfContentSummary, _PdfXObjects]
@dataclass(frozen=True, slots=True)
class _PdfWalk:
frontier: tuple[_PdfDrawing, ...]
seen: frozenset[tuple[int, int]]
summaries: Mapping[int, _PdfContentSummary | None]
draws: int
def _pdf_xobjects(container: Mapping[str, object], inherited: _PdfXObjects) -> _PdfXObjects:
resources: Final = _pdf_dictionary(_pdf_value(container, "/Resources"))
if resources is None:
return inherited
return _pdf_dictionary(_pdf_value(resources, "/XObject"))
_PDF_WHITESPACE: Final = rb"\x00\t\n\x0c\r "
_PDF_TOKEN_END: Final = _PDF_WHITESPACE + rb"()<>\[\]{}/%"
_PDF_XOBJECT_DRAW: Final = re.compile(
rb"/([^" + _PDF_TOKEN_END + rb"]*)(?:[" + _PDF_WHITESPACE + rb"]|%[^\r\n]*[\r\n])+Do(?![^" + _PDF_TOKEN_END + rb"])"
)
_PDF_INLINE_IMAGE: Final = re.compile(rb"(?<![^" + _PDF_WHITESPACE + rb"])BI(?![^" + _PDF_WHITESPACE + rb"/])")
_PDF_NAME_ESCAPE: Final = re.compile(rb"#([0-9A-Fa-f]{2})")
_PDF_NAME_CHARSETS: Final = ("utf-8", "gbk")
def _decoded(raw: bytes, charset: str) -> str | None:
try:
return raw.decode(charset)
except UnicodeDecodeError:
return None
def _pdf_name(escaped: bytes) -> str:
raw: Final = _PDF_NAME_ESCAPE.sub(lambda escape: bytes.fromhex(escape[1].decode("ascii")), escaped)
decoded: Final = (_decoded(raw, charset) for charset in _PDF_NAME_CHARSETS)
return "/" + next((name for name in decoded if name is not None), raw.decode("latin-1"))
def _pdf_content_summary(content: _PdfContent) -> _PdfContentSummary:
data: Final = content.get_data()
draws: Final = islice(_PDF_XOBJECT_DRAW.finditer(data), PDF_MAX_XOBJECT_DRAWS_PER_PAGE + 1)
return _PdfContentSummary(
drawn_names=tuple(_pdf_name(draw[1]) for draw in draws),
draws_inline_image=_PDF_INLINE_IMAGE.search(data) is not None,
)
def _pdf_form_summary(form: Mapping[str, object]) -> _PdfContentSummary | None:
from pypdf.generic import StreamObject
if not isinstance(form, StreamObject):
return None
return _pdf_content_summary(cast(_PdfContent, form)) # cast-ok: pypdf is untyped
def _pdf_drawn_xobjects(drawing: _PdfDrawing) -> tuple[_PdfXObjectUse, ...]:
summary, xobjects = drawing
if xobjects is None:
return ()
resolved: Final = (_pdf_dictionary(_pdf_value(xobjects, name)) for name in summary.drawn_names)
return tuple((xobject, xobjects) for xobject in resolved if xobject is not None)
def _pdf_drawing_key(use: _PdfXObjectUse) -> tuple[int, int]:
xobject, inherited = use
return id(xobject), id(_pdf_xobjects(xobject, inherited))
def _pdf_next_walk(walk: _PdfWalk) -> _PdfWalk | None:
draws: Final = walk.draws + sum(len(summary.drawn_names) for summary, _ in walk.frontier)
if draws > PDF_MAX_XOBJECT_DRAWS_PER_PAGE:
return None
drawn: Final = tuple(chain.from_iterable(map(_pdf_drawn_xobjects, walk.frontier)))
if any(_pdf_value(xobject, "/Subtype") == "/Image" for xobject, _ in drawn):
return None
forms: Final = {_pdf_drawing_key(use): use for use in drawn if _pdf_value(use[0], "/Subtype") == "/Form"}
unseen: Final = tuple(forms[key] for key in forms.keys() - walk.seen)
unparsed: Final = {id(form): form for form, _ in unseen if id(form) not in walk.summaries}
summaries: Final = MappingProxyType(
{**walk.summaries, **{key: _pdf_form_summary(form) for key, form in unparsed.items()}}
)
drawings: Final = ((summaries[id(form)], _pdf_xobjects(form, inherited)) for form, inherited in unseen)
return _PdfWalk(
frontier=tuple((summary, xobjects) for summary, xobjects in drawings if summary is not None),
seen=walk.seen | frozenset(forms),
summaries=summaries,
draws=draws,
)
def _pdf_page_draws_raster_image(page: _PdfPage) -> bool:
contents: Final = page.get_contents()
page_dictionary: Final = _pdf_dictionary(page)
if contents is None or page_dictionary is None:
return False
walk: _PdfWalk = _PdfWalk( # rebind-ok: breadth-first walk, recursion is banned
frontier=((_pdf_content_summary(contents), _pdf_xobjects(page_dictionary, None)),),
seen=frozenset(),
summaries=MappingProxyType({}),
draws=0,
)
for _ in range(PDF_MAX_FORM_XOBJECT_DEPTH):
if not walk.frontier:
return False
if any(summary.draws_inline_image for summary, _ in walk.frontier):
return True
if (next_walk := _pdf_next_walk(walk)) is None:
return True
walk = next_walk
return bool(walk.frontier)
def _openai_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
if _pdf_page_draws_raster_image(page):
return _claude_pdf_page_tokens(page, count_function)
text: Final = page.extract_text() or ""
return count_function(text) + OPENAI_PDF_TEXT_PAGE_FRAMING_TOKENS if text.strip() else 0
def _claude_pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction) -> int:
return count_function(page.extract_text() or "") + _anthropic_rendered_page_image_tokens(
float(page.mediabox.width), float(page.mediabox.height)
)
def _pdf_page_tokens(page: _PdfPage, count_function: TokenCounterFunction, pdf_pricing: PdfPagePricing) -> int:
match pdf_pricing:
case "openai":
return _openai_pdf_page_tokens(page, count_function)
case "claude":
return _claude_pdf_page_tokens(page, count_function)
def _count_inline_pdf_tokens(
data_url: str,
count_function: TokenCounterFunction,
pdf_pricing: PdfPagePricing = "claude",
) -> int | None:
if not data_url.startswith(PDF_DATA_URL_PREFIX):
return None
try:
@ -1262,12 +1526,10 @@ def _count_inline_pdf_tokens(data_url: str, count_function: TokenCounterFunction
verbose_logger.debug("pypdf is not installed, so the PDF document is priced like one image")
return None
try:
reader: Final = PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
return sum(
count_function(page.extract_text() or "")
+ _anthropic_rendered_page_image_tokens(float(page.mediabox.width), float(page.mediabox.height))
for page in reader.pages
reader: Final = cast( # cast-ok: pypdf's PdfReader satisfies _PdfReader, typed only where pypdf is installed
_PdfReader, PdfReader(io.BytesIO(base64.b64decode(data_url[len(PDF_DATA_URL_PREFIX) :])))
)
return sum(_pdf_page_tokens(page, count_function, pdf_pricing) for page in reader.pages)
except Exception as e:
verbose_logger.debug("Could not read the PDF document's pages (%s), so it is priced like one image", e)
return None
@ -1277,8 +1539,9 @@ def _count_opaque_document_tokens(
data_url: str,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function)
pdf_tokens: Final = _count_inline_pdf_tokens(data_url, count_function, pdf_pricing)
if pdf_tokens is not None:
return pdf_tokens
return calculate_img_tokens(
@ -1293,6 +1556,7 @@ def _count_document_tokens(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
source: Final = document["source"]
metadata_tokens: Final = sum(
@ -1305,10 +1569,10 @@ def _count_document_tokens(
if isinstance(content, str):
return metadata_tokens + count_function(content)
return metadata_tokens + _count_content_list(
count_function, content, use_default_image_token_count, default_token_count
count_function, content, use_default_image_token_count, default_token_count, pdf_pricing
)
return metadata_tokens + _count_opaque_document_tokens(
_anthropic_image_source_data(source), count_function, use_default_image_token_count
_anthropic_image_source_data(source), count_function, use_default_image_token_count, pdf_pricing
)
@ -1316,6 +1580,7 @@ def _count_file_tokens(
file_value: object,
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""An OpenAI `file` block is the chat-completions spelling of a document, so it prices like one."""
if not isinstance(file_value, Mapping):
@ -1325,7 +1590,9 @@ def _count_file_tokens(
name_tokens: Final = count_function(filename) if isinstance(filename, str) and filename else 0
if not isinstance(file_data, str) or not file_data:
return name_tokens
return name_tokens + _count_opaque_document_tokens(file_data, count_function, use_default_image_token_count)
return name_tokens + _count_opaque_document_tokens(
file_data, count_function, use_default_image_token_count, pdf_pricing
)
def _count_anthropic_content(
@ -1333,6 +1600,7 @@ def _count_anthropic_content(
count_function: TokenCounterFunction,
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""
Count tokens in Anthropic-specific content blocks (tool_use, tool_result, etc.).
@ -1367,6 +1635,7 @@ def _count_anthropic_content(
field_value,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif isinstance(field_value, dict):
tokens += count_function(str(field_value))
@ -1393,6 +1662,7 @@ def _count_content_list(
],
use_default_image_token_count: bool,
default_token_count: int | None,
pdf_pricing: PdfPagePricing = "claude",
) -> int:
"""Recursively count tokens from a list of content blocks."""
try:
@ -1417,12 +1687,14 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] == "file":
num_tokens += _count_file_tokens(
c.get("file"),
count_function,
use_default_image_token_count,
pdf_pricing,
)
elif c["type"] in ("tool_use", "tool_result", "search_result"):
num_tokens += _count_anthropic_content(
@ -1430,6 +1702,7 @@ def _count_content_list(
count_function,
use_default_image_token_count,
default_token_count,
pdf_pricing,
)
elif c["type"] in ("thinking", "redacted_thinking"):
# Claude extended thinking content block
@ -1552,12 +1825,13 @@ def _format_single_function_definition(function: Mapping[str, object]) -> tuple[
def _format_object_parameters(parameters, indent):
properties: Final = parameters.get("properties")
if not properties:
if not isinstance(properties, Mapping) or not properties:
return ""
required_params: Final = parameters.get("required", [])
required: Final = parameters.get("required")
required_params: Final = tuple(required) if isinstance(required, list) else ()
lines: Final = []
for key, props in properties.items():
description = props.get("description")
description = props.get("description") if isinstance(props, Mapping) else None
if description:
lines.append(f"// {description}")
question = "?"
@ -1568,19 +1842,21 @@ def _format_object_parameters(parameters, indent):
def _format_type(props, indent):
if not isinstance(props, Mapping):
return "any"
type: Final = props.get("type")
enum: Final = props.get("enum")
if type == "string":
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "string"
elif type == "array":
# items is required, OpenAI throws an error if it's missing
return f"{_format_type(props['items'], indent)}[]"
return f"{_format_type(props.get('items'), indent)}[]"
elif type == "object":
return f"{{\n{_format_object_parameters(props, indent + 2)}\n}}"
elif type in ["integer", "number"]:
if "enum" in props:
return " | ".join([f'"{item}"' for item in props["enum"]])
if isinstance(enum, list):
return " | ".join([f'"{item}"' for item in enum])
return "number"
elif type == "boolean":
return "boolean"

View file

@ -14,6 +14,7 @@ import json
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Optional
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
@ -28,6 +29,8 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
class _A2ATextPart(TypedDict, total=False):
"""The subset of an A2A message part this handler reads text from."""
@ -333,8 +336,8 @@ class A2AGuardrailHandler(BaseTranslation):
obj = item
elif isinstance(item, str):
try:
obj = json.loads(item.strip())
except (json.JSONDecodeError, TypeError):
obj = _JSON_OBJECT.validate_python(json.loads(item.strip()))
except (json.JSONDecodeError, TypeError, ValidationError):
continue
else:
continue

View file

@ -18,9 +18,9 @@ from copy import deepcopy
from dataclasses import dataclass
from itertools import chain, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, overload, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast, overload, runtime_checkable
from pydantic import TypeAdapter, ValidationError
from pydantic import ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import ReadOnly, TypedDict, assert_never
from litellm._logging import verbose_proxy_logger
@ -349,14 +349,40 @@ def _tool_call_shapes(tool_calls: Sequence[object]) -> tuple[_ToolCallShape, ...
)
class _AnthropicSSEDelta(TypedDict, total=False):
type: ReadOnly[str]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSETextDelta(TypedDict, total=False):
type: ReadOnly[object]
text: ReadOnly[str]
stop_reason: ReadOnly[str | None]
class _AnthropicSSEEvent(TypedDict, total=False):
delta: ReadOnly[_AnthropicSSEDelta]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSETextDeltaEvent(TypedDict, total=False):
delta: ReadOnly[_SSETextDelta]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSEStopDelta(TypedDict, total=False):
stop_reason: ReadOnly[object]
@with_config(ConfigDict(extra="ignore", strict=True))
class _SSEStopDeltaEvent(TypedDict, total=False):
delta: ReadOnly[_SSEStopDelta]
_SSE_TEXT_DELTA_EVENT: Final = TypeAdapter(_SSETextDeltaEvent)
_SSE_STOP_DELTA_EVENT: Final = TypeAdapter(_SSEStopDeltaEvent)
_SSEEventT = TypeVar("_SSEEventT")
def _parse_sse_event(adapter: TypeAdapter[_SSEEventT], event_type: str, data_line: str) -> _SSEEventT | None:
try:
return adapter.validate_python(json.loads(data_line))
except json.JSONDecodeError:
verbose_proxy_logger.warning("Skipping a %s SSE event that is not valid JSON", event_type)
except ValidationError:
verbose_proxy_logger.warning("Skipping a %s SSE event with an unexpected shape", event_type)
return None
class AnthropicMessagesHandler(BaseTranslation):
@ -1806,13 +1832,10 @@ class AnthropicMessagesHandler(BaseTranslation):
# Only process content_block_delta events
if event_type == "content_block_delta" and data_line:
try:
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
if delta.get("type") == "text_delta":
text += delta.get("text", "")
except json.JSONDecodeError:
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
data = _parse_sse_event(_SSE_TEXT_DELTA_EVENT, event_type, data_line)
delta: _SSETextDelta = {} if data is None else data.get("delta", {})
if delta.get("type") == "text_delta":
text += delta.get("text", "")
except Exception as e:
verbose_proxy_logger.error("Error extracting text from SSE: %s", e)
@ -1869,14 +1892,11 @@ class AnthropicMessagesHandler(BaseTranslation):
# Check for message_delta event with stop_reason
if event_type == "message_delta" and data_line:
try:
data: _AnthropicSSEEvent = json.loads(data_line)
delta: _AnthropicSSEDelta = data.get("delta", {})
stop_reason: str | None = delta.get("stop_reason")
if stop_reason is not None:
return True
except json.JSONDecodeError:
verbose_proxy_logger.warning("Failed to parse JSON from SSE data: %s", data_line)
data = _parse_sse_event(_SSE_STOP_DELTA_EVENT, event_type, data_line)
delta: _SSEStopDelta = {} if data is None else data.get("delta", {})
stop_reason: object = delta.get("stop_reason")
if stop_reason is not None:
return True
except Exception as e:
verbose_proxy_logger.error("Error checking streaming end in SSE: %s", e)

View file

@ -6,6 +6,7 @@ import copy
import json
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timezone
from types import MappingProxyType
from typing import Annotated, Any, ClassVar, Final, Literal, TypeVar
@ -1774,6 +1775,48 @@ def anthropic_request_errors_on_thinking_binding_mismatch(request_body: object)
return binding is not None and binding.get("prefix_mismatch_behavior") == "error"
_CLAUDE_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_CLAUDE_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def is_claude_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _CLAUDE_STRICT_TOOLS_REJECTION.search(error_text) is not None
@dataclass(frozen=True, slots=True)
class ToolsWithoutStrict:
tools: tuple[Mapping[str, object], ...]
dropped_count: int
def tools_without_strict(tools: object) -> ToolsWithoutStrict | None:
try:
validated: Final = _CLAUDE_TOOLS_ADAPTER.validate_python(tools)
except ValidationError:
return None
dropped_count: Final = sum(tool.get("strict") is True for tool in validated)
if dropped_count == 0:
return None
return ToolsWithoutStrict(
tools=tuple({key: value for key, value in tool.items() if key != "strict"} for tool in validated),
dropped_count=dropped_count,
)
def warn_claude_strict_tools_dropped(dropped_count: int) -> None:
litellm.verbose_logger.warning(
"Claude rejected the request's strict schemas, so litellm removed `strict` from every tool (%d had "
"strict true) and sent the request again because drop_params is set.",
dropped_count,
)
def strip_thinking_blocks_from_anthropic_messages(messages: Sequence[object]) -> list[object]:
"""
Return a new message list with thinking / redacted_thinking content blocks removed

View file

@ -160,6 +160,7 @@ async def prepare_context_managed_request(
"""Apply client compaction history, then optional context_management polyfill."""
from litellm.llms.anthropic.messages.context_management.editors.compact import (
apply_client_compaction_block_history,
slice_around_compaction_block,
)
# Skip the client-history pre-processing when a ``compact_20260112``
@ -175,7 +176,7 @@ async def prepare_context_managed_request(
if polyfill_will_run:
history_result: PolyfillResult | None = None
working_messages: _AnthropicMessages = messages
working_messages: _AnthropicMessages = list(slice_around_compaction_block(messages)[0])
working_system: _AnthropicSystem = system
else:
history_result = apply_client_compaction_block_history(

View file

@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.types.llms.anthropic import AppliedEdit
from ..convertible_blocks import drop_non_string_tool_result_texts
from .constants import CLEAR_TOOL_USES_EDIT_TYPE, COMPACT_EDIT_TYPE
from .editors import (
CheckedEdit,
@ -103,7 +104,7 @@ async def apply_context_management(
Every ``clear_tool_uses_20250919`` edit is validated before any editor runs, so a malformed one raises its 400
before a ``compact_20260112`` summary call is paid for, or runs without its malformed fields under
``drop_invalid_edits``.
``drop_invalid_edits``, which also drops non-string ``tool_result`` text first, as the provider never receives it.
The dispatcher is async so async editors (``compact_20260112``) can
``await`` the configured summarization model. Sync editors run in a
@ -119,7 +120,7 @@ async def apply_context_management(
raise first_error
well_formed_edits: Final = tuple(check.edit for check in checked)
current_messages = messages
current_messages = list(drop_non_string_tool_result_texts(messages, drop_invalid_edits))
current_system = system
aggregated_applied: Final[list[AppliedEdit]] = []
aggregated_compaction_block = None

View file

@ -39,6 +39,21 @@ if TYPE_CHECKING:
T = TypeVar("T")
# api.anthropic.com, measured 2026-10-19: text results of 1 to 46 bytes are never cleared or counted toward keep
_TINY_RESULT_BYTES: Final = 46
_TOKEN_COUNTER_BLOCK_TYPES: Final = frozenset(
{
"text",
"image_url",
"image",
"document",
"file",
"tool_use",
"tool_result",
"thinking",
"redacted_thinking",
"tool_reference",
"search_result",
}
)
ToolNames = Literal["all"] | frozenset[str]
@ -52,16 +67,19 @@ class _ClearAtLeast(BaseModel):
class _ClearToolUsesFields(BaseModel):
model_config = ConfigDict(frozen=True)
trigger: Mapping[str, object] | None = None
keep: Mapping[str, object] | None = None
clear_at_least: _ClearAtLeast | None = None
exclude_tools: tuple[str, ...] | None = None
clear_tool_inputs: bool | tuple[str, ...] | None = None
_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
# api.anthropic.com, measured 2026-10-19: its wording for the two shapes pydantic words differently
# api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
_API_MESSAGES: Final = MappingProxyType(
{
"tuple_type": "Input should be a valid array",
"dict_type": "Input should be an object",
"model_type": "Input does not match the expected shape.",
}
)
@ -178,10 +196,12 @@ def _cleared_result(block: Mapping[str, object]) -> dict[str, object]:
def _freed_tokens(model: str, result: Mapping[str, object], use: Mapping[str, object], clears_input: bool) -> int:
"""Tokens that clearing this use frees: its result, and its input when ``clear_tool_inputs`` covers it."""
before: Final = [result, use] if clears_input else [result]
after: Final = [_cleared_result(result), {**use, "input": {}}] if clears_input else [_cleared_result(result)]
before: Final[Sequence[Mapping[str, object]]] = (result, use) if clears_input else (result,)
after: Final[Sequence[Mapping[str, object]]] = (
(_cleared_result(result), {**use, "input": {}}) if clears_input else (_cleared_result(result),)
)
counts: Final = tuple(
litellm.token_counter(model=model, messages=[{"role": "user", "content": blocks}]) for blocks in (before, after)
_count_tokens(model, [{"role": "user", "content": blocks}], None) for blocks in (before, after)
)
return max(counts[0] - counts[1], 0)
@ -199,6 +219,35 @@ def _select_in_batches(candidates: Sequence[str], tokens: Mapping[str, int], cle
return tuple(candidates[:batch_end])
def _without_uncountable_blocks(block: Mapping[str, object]) -> Mapping[str, object]:
nested: Final = _parse(_BLOCKS, block.get("content"))
if nested is None:
return block
return {**block, "content": [inner for inner in nested if inner.get("type") in _TOKEN_COUNTER_BLOCK_TYPES]}
def _countable_message(message: Mapping[str, object]) -> Mapping[str, object]:
blocks: Final = _parse(_BLOCKS, message.get("content"))
if blocks is None:
return message
return {
**message,
"content": [
_without_uncountable_blocks(block) for block in blocks if block.get("type") in _TOKEN_COUNTER_BLOCK_TYPES
],
}
def _count_tokens(
model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None
) -> int:
return litellm.token_counter(
model=model,
messages=[_countable_message(message) for message in messages],
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
def _trigger_met(
trigger: dict[str, object],
model: str,
@ -218,11 +267,7 @@ def _trigger_met(
if not isinstance(threshold, int):
threshold = DEFAULT_INPUT_TOKENS_TRIGGER
message_tokens: Final = litellm.token_counter(
model=model,
messages=messages,
tools=cast("list[ChatCompletionToolParam] | None", tools),
)
message_tokens: Final = _count_tokens(model, messages, tools)
if message_tokens > threshold:
return True, message_tokens
current_tokens: Final = message_tokens + count_system_tokens(model, system)
@ -375,17 +420,9 @@ def apply_clear_tool_uses_20250919(
[_message_without_inputs(message, ids_to_clear, input_tools) for message in edited] if input_tools else edited
)
tokens_before: Final = (
counted_tokens
if counted_tokens is not None
else litellm.token_counter(
model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
)
tokens_before: Final = counted_tokens if counted_tokens is not None else _count_tokens(model, kept, tools)
kept_after, _ = history_after_client_compaction(final_messages, system)
tokens_after: Final = litellm.token_counter(
model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
)
tokens_after: Final = _count_tokens(model, kept_after, tools)
cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
return final_messages, {

View file

@ -137,7 +137,7 @@ def _find_latest_compaction_index(
return None, None
def _slice_around_compaction_block(
def slice_around_compaction_block(
messages: Sequence[_MsgT],
) -> tuple[Sequence[_MsgT | dict[str, object]], dict[str, object] | None]:
"""Apply Anthropic's "drop everything before the compaction block" rule.
@ -194,7 +194,7 @@ def history_after_client_compaction(
system: str | Sequence[Mapping[str, object]] | None,
) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
"""Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
sliced, block = _slice_around_compaction_block(messages)
sliced, block = slice_around_compaction_block(messages)
if block is None:
return list(messages), system
tail: Final = _strip_compaction_blocks(sliced)
@ -615,7 +615,7 @@ def apply_client_compaction_block_history(
tail is forwarded unchanged (with compaction blocks stripped) so recent
turns the summary does not cover are preserved.
"""
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
if prior_compaction_block is None:
return None
@ -686,7 +686,7 @@ async def apply_compact_20260112(
# opt-in gate below so that even when summarization is disabled we still
# strip Anthropic-only ``compaction`` blocks from messages going to
# non-Anthropic backends (which would reject them).
effective_messages, prior_compaction_block = _slice_around_compaction_block(messages)
effective_messages, prior_compaction_block = slice_around_compaction_block(messages)
prior_summary_text: Final = prior_compaction_block.get("content") if prior_compaction_block else None
augmented_system: str | list[dict[str, object]] | None = system
if isinstance(prior_summary_text, str) and prior_summary_text:

View file

@ -10,11 +10,15 @@ from typing import Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
supports_openai_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.asyncify import (
run_async_function, # pyright: ignore[reportUnknownVariableType] # untyped helper
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.prompt_templates.common_utils import (
cache_control_as_prompt_cache_breakpoint,
prompt_cache_breakpoint_for_wire,
@ -28,14 +32,19 @@ from litellm.types.llms.anthropic import (
AnthropicMessagesRequest,
AnthropicOutputConfig,
AnthropicOutputSchema,
ContextManagementResponse,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import CountContextManagement
from litellm.utils import ProviderConfigManager
from ..adapters.handler import request_safeguards_evaluator
from ..adapters.handler import context_management_explicitly_dropped, request_safeguards_evaluator
from ..context_management import AnthropicContextManagementError, PolyfillResult, apply_context_management
from ..context_management.constants import COMPACT_EDIT_TYPE
from ..context_management.dispatcher import normalize_spec
from ..convertible_blocks import reject_malformed_tool_result_text
from ..safeguards import with_safeguard_results
from ..utils import litellm_logging_obj_from_kwargs, local_model_name
@ -45,10 +54,111 @@ from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
AnthropicRequestMessages: TypeAlias = list[AllAnthropicMessageValues] | list[dict[str, object]]
BlockRewrite: TypeAlias = Callable[[Mapping[str, object]], Mapping[str, object]]
_BLOCK_LIST: Final = TypeAdapter(list[dict[str, object]])
_SYSTEM_ITEMS: Final = TypeAdapter(list[object])
_SPEC: Final = TypeAdapter(dict[str, object])
_SplitEdits: TypeAlias = tuple[dict[str, object] | None, Sequence[Mapping[str, object]]]
AnthropicContextManagementSpec: TypeAlias = dict[str, object] | list[dict[str, object]] | None
_ADAPTER: Final = LiteLLMAnthropicToResponsesAPIAdapter()
def _split_context_management(
spec: AnthropicContextManagementSpec,
kwargs: Mapping[str, object],
) -> _SplitEdits:
"""Return (the edits the Responses API runs natively, the edits the gateway polyfill runs).
The Responses API compacts natively (``compact_20260112`` becomes its ``compaction``); any other
edit, such as ``clear_tool_uses_20250919``, has no Responses API form and runs in the polyfill, as
it does on the chat completions path. An OpenAI-style list spec is not Anthropic's shape and stays
unsent, as it was before the polyfill ran here.
"""
if not isinstance(spec, dict):
return None, ()
if not spec or context_management_explicitly_dropped(kwargs.get("additional_drop_params")):
return spec, ()
edits: Final = normalize_spec(spec) or []
native: Final = [edit for edit in edits if edit.get("type") == COMPACT_EDIT_TYPE]
return ({"edits": native} if native else None), tuple(
edit for edit in edits if edit.get("type") != COMPACT_EDIT_TYPE
)
def _system_blocks(system: object) -> str | Sequence[Mapping[str, object]] | None:
if system is None or isinstance(system, str):
return system
try:
items: Final = _SYSTEM_ITEMS.validate_python(system)
except ValidationError:
return None
return tuple(block for block in map(as_str_mapping, items) if block is not None)
async def _polyfilled(
*,
model: str,
messages: AnthropicRequestMessages,
tools: Sequence[AllAnthropicToolsValues | Mapping[str, object]] | None,
system: object,
edits: Sequence[Mapping[str, object]],
kwargs: Mapping[str, object],
) -> PolyfillResult | None:
"""Run the edits the Responses API can't run natively; raise like the chat completions path does."""
if not edits:
return None
reject_malformed_tool_result_text(messages, model, kwargs)
drop_invalid_edits: Final = drops_unsupported_params(kwargs.get("drop_params"))
blocks: Final = _system_blocks(system)
try:
return await apply_context_management(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=_BLOCK_LIST.validate_python(tools) if tools else None,
system=blocks if blocks is None or isinstance(blocks, str) else _BLOCK_LIST.validate_python(blocks),
context_management_spec={"edits": [dict(edit) for edit in edits]},
drop_invalid_edits=drop_invalid_edits,
)
except AnthropicContextManagementError:
raise
except Exception as e:
verbose_logger.exception("context_management polyfill failed on the Responses API path: %s", e)
raise AnthropicContextManagementError(
status_code=500, message=f"context_management polyfill failed: {e}"
) from e
async def count_view_messages(
*,
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]]:
"""The messages /v1/messages sends the Responses API once the polyfill has run the edits it can't run natively."""
if context_management is None or not isinstance(context_management.spec, dict):
return messages
_, edits = _split_context_management(_SPEC.validate_python(context_management.spec), {})
result: Final = await _polyfilled(
model=model,
messages=_BLOCK_LIST.validate_python(messages),
tools=tools,
system=system,
edits=edits,
kwargs={"drop_params": context_management.drop_invalid_edits},
)
return messages if result is None else result.messages
def _with_applied_edits(
response: AnthropicMessagesResponse, polyfill_result: PolyfillResult | None
) -> AnthropicMessagesResponse:
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
if not applied_edits:
return response
return {**response, "context_management": ContextManagementResponse(applied_edits=list(applied_edits))}
def _record_bridged_response(responses_kwargs: Mapping[str, object], result: ResponsesAPIResponse) -> None:
logging_obj: Final = litellm_logging_obj_from_kwargs(responses_kwargs)
if logging_obj is not None:
@ -281,7 +391,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -296,11 +406,15 @@ class LiteLLMMessagesToResponsesAPIHandler:
output_format: AnthropicOutputSchema | None = None,
**kwargs: object,
) -> AnthropicMessagesResponse | AsyncIterator[bytes]:
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
polyfill_result: Final = await _polyfilled(
model=model, messages=messages, tools=tools, system=system, edits=polyfilled_edits, kwargs=kwargs
)
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -325,13 +439,16 @@ class LiteLLMMessagesToResponsesAPIHandler:
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
safeguards_evaluator=safeguards_evaluator,
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
served: Final = await with_safeguard_results(_ADAPTER.translate_response(result), safeguards_evaluator)
served: Final = await with_safeguard_results(
_with_applied_edits(_ADAPTER.translate_response(result), polyfill_result), safeguards_evaluator
)
_record_bridged_response(responses_kwargs, result)
return served
@ -340,7 +457,7 @@ class LiteLLMMessagesToResponsesAPIHandler:
max_tokens: int,
messages: AnthropicRequestMessages,
model: str,
context_management: dict[str, object] | None = None,
context_management: AnthropicContextManagementSpec = None,
metadata: dict[str, object] | None = None,
output_config: AnthropicOutputConfig | None = None,
stop_sequences: list[str] | None = None,
@ -382,11 +499,26 @@ class LiteLLMMessagesToResponsesAPIHandler:
)
# Sync path
native_context_management, polyfilled_edits = _split_context_management(context_management, kwargs)
ran: Final[object] = ( # pyright: ignore[reportUnknownVariableType] # run_async_function is untyped
run_async_function(
_polyfilled,
model=model,
messages=messages,
tools=tools,
system=system,
edits=polyfilled_edits,
kwargs=kwargs,
)
if polyfilled_edits
else None
)
polyfill_result: Final = ran if isinstance(ran, PolyfillResult) else None
responses_kwargs: Final = _build_responses_kwargs(
max_tokens=max_tokens,
messages=messages,
messages=polyfill_result.messages if polyfill_result is not None else messages,
model=model,
context_management=context_management,
context_management=native_context_management,
metadata=metadata,
output_config=output_config,
stop_sequences=stop_sequences,
@ -409,10 +541,11 @@ class LiteLLMMessagesToResponsesAPIHandler:
responses_stream=result,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
litellm_logging_obj=litellm_logging_obj_from_kwargs(responses_kwargs),
applied_edits=polyfill_result.applied_edits_for_response() if polyfill_result else None,
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}")
return _ADAPTER.translate_response(result)
return _with_applied_edits(_ADAPTER.translate_response(result), polyfill_result)

View file

@ -2,7 +2,7 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
import anyio
@ -27,6 +27,7 @@ from litellm.responses.streaming_iterator import (
partial_responses_incomplete_event_on_disconnect,
stream_error_status_and_message,
)
from litellm.types.llms.anthropic import AppliedEdit, ContextManagementResponse
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage
from litellm.types.llms.base import LiteLLMBaseModel
@ -129,8 +130,10 @@ class AnthropicResponsesStreamWrapper:
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
applied_edits: Sequence[AppliedEdit] | None = None,
) -> None:
self.responses_stream = responses_stream
self._applied_edits: Final = tuple(applied_edits or ())
self.model = model
self.litellm_logging_obj = litellm_logging_obj
self.safeguards_evaluator: Final = safeguards_evaluator
@ -445,6 +448,11 @@ class AnthropicResponsesStreamWrapper:
"type": "message_delta",
"delta": message_delta_payload,
"usage": dict(anthropic_usage),
**(
{"context_management": ContextManagementResponse(applied_edits=list(self._applied_edits))}
if self._applied_edits
else {}
),
}
)
self._chunk_queue.append({"type": "message_stop"})

View file

@ -1,7 +1,12 @@
from typing import TYPE_CHECKING, Any, Final
import math
from collections.abc import Mapping
from typing import TYPE_CHECKING, Annotated, Final, TypeAlias
import httpx
from pydantic import BeforeValidator, ConfigDict, TypeAdapter, ValidationError, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.image_generation.transformation import (
@ -18,6 +23,67 @@ if TYPE_CHECKING:
from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
_TOKEN_COUNT_ADAPTER: Final[TypeAdapter[int | float]] = TypeAdapter(int | float)
def _token_count_or_none(value: object) -> int | float | None:
if value is None:
return None
try:
count: Final = _TOKEN_COUNT_ADAPTER.validate_python(value)
except ValidationError:
verbose_logger.warning("MAI image usage has a token count that is not a number, counting it as absent")
return None
if not math.isfinite(count):
verbose_logger.warning("MAI image usage has a token count that is not finite, counting it as absent")
return None
return count
_TokenCount: TypeAlias = Annotated[int | float | None, BeforeValidator(_token_count_or_none)]
@with_config(ConfigDict(extra="allow"))
class _MAIImageUsageDetails(TypedDict, total=False):
text_tokens: ReadOnly[NotRequired[_TokenCount]]
image_tokens: ReadOnly[NotRequired[_TokenCount]]
@with_config(ConfigDict(extra="allow"))
class _MAIImageUsage(TypedDict, total=False):
input_tokens_details: ReadOnly[NotRequired[object]]
num_input_text_tokens: ReadOnly[NotRequired[_TokenCount]]
num_input_image_tokens: ReadOnly[NotRequired[_TokenCount]]
input_tokens: ReadOnly[NotRequired[_TokenCount]]
output_tokens: ReadOnly[NotRequired[_TokenCount]]
num_output_tokens: ReadOnly[NotRequired[_TokenCount]]
output_image_tokens: ReadOnly[NotRequired[_TokenCount]]
total_tokens: ReadOnly[NotRequired[_TokenCount]]
@with_config(ConfigDict(extra="allow"))
class _MAIImageResponse(TypedDict, total=False):
usage: ReadOnly[NotRequired[object]]
_MAI_IMAGE_USAGE_DETAILS_ADAPTER: Final[TypeAdapter[_MAIImageUsageDetails]] = TypeAdapter(_MAIImageUsageDetails)
_MAI_IMAGE_USAGE_ADAPTER: Final[TypeAdapter[_MAIImageUsage]] = TypeAdapter(_MAIImageUsage)
_MAI_RAW_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
def _raw_usage_or_none(usage: object) -> Mapping[str, object] | None:
if usage is None:
return None
try:
return _MAI_RAW_USAGE_ADAPTER.validate_python(usage)
except ValidationError:
verbose_logger.warning("MAI image usage is not an object, counting it as absent")
return None
_MAI_IMAGE_RESPONSE_ADAPTER: Final[TypeAdapter[_MAIImageResponse]] = TypeAdapter(_MAIImageResponse)
class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"""Azure AI Foundry MAI image generation (e.g. MAI-Image-2.5)."""
@ -78,9 +144,10 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
return "maiimage" in model_normalized
@staticmethod
def normalize_mai_image_usage(usage: dict[str, Any] | None) -> dict[str, Any]:
def normalize_mai_image_usage(usage: object) -> Mapping[str, object]:
"""Map Azure MAI usage fields to OpenAI ImageUsage schema."""
if usage is None:
raw_usage: Final = _raw_usage_or_none(usage)
if raw_usage is None:
return {
"input_tokens": 0,
"input_tokens_details": {"image_tokens": 0, "text_tokens": 0},
@ -88,10 +155,13 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
"total_tokens": 0,
}
normalized_usage: Final = dict(usage)
input_tokens_details = normalized_usage.get("input_tokens_details")
if not isinstance(input_tokens_details, dict):
input_tokens_details = {}
normalized_usage: Final[_MAIImageUsage] = _MAI_IMAGE_USAGE_ADAPTER.validate_python(raw_usage)
input_tokens_details_value: Final = normalized_usage.get("input_tokens_details")
input_tokens_details: Final[_MAIImageUsageDetails] = (
_MAI_IMAGE_USAGE_DETAILS_ADAPTER.validate_python(input_tokens_details_value)
if isinstance(input_tokens_details_value, dict)
else {}
)
text_tokens = normalized_usage.get("num_input_text_tokens")
if text_tokens is None:
@ -121,18 +191,16 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
if total_tokens is None:
total_tokens = input_tokens + output_tokens
normalized_usage.update(
{
"input_tokens": input_tokens,
"input_tokens_details": {
"image_tokens": image_tokens,
"text_tokens": text_tokens,
},
"output_tokens": output_tokens,
"total_tokens": total_tokens,
}
)
return normalized_usage
return {
**raw_usage,
"input_tokens": input_tokens,
"input_tokens_details": {
"image_tokens": image_tokens,
"text_tokens": text_tokens,
},
"output_tokens": output_tokens,
"total_tokens": total_tokens,
}
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
return ["n", "size"]
@ -252,12 +320,18 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig):
json_mode: bool | None = None,
) -> ImageResponse:
try:
response: Final = raw_response.json()
response_json: Final = _MAI_IMAGE_RESPONSE_ADAPTER.validate_python(raw_response.json())
except Exception:
raise OpenAIError(message=raw_response.text, status_code=raw_response.status_code)
if "usage" in response:
response["usage"] = self.normalize_mai_image_usage(response.get("usage"))
response: Final = (
{
**response_json,
"usage": self.normalize_mai_image_usage(response_json["usage"]),
}
if "usage" in response_json
else dict(response_json)
)
logging_obj.post_call(
input=request_data.get("prompt", ""),

View file

@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final
import httpx
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
@ -201,11 +202,23 @@ class BaseAnthropicMessagesConfig(ABC):
anthropic_request_errors_on_thinking_binding_mismatch,
is_anthropic_invalid_thinking_block_error,
is_anthropic_thinking_binding_mismatch_error,
is_claude_strict_tools_rejection,
tools_without_strict,
validated_object_mapping,
)
if e.response.status_code != 400:
return False
error_text: Final[str] = e.response.text
body: Final = validated_object_mapping(request_body)
drops_rejected_strict_tools: Final = (
drops_unsupported_params(litellm_params.get("drop_params"))
and is_claude_strict_tools_rejection(e.response.status_code, error_text)
and body is not None
and tools_without_strict(body.get("tools")) is not None
)
if drops_rejected_strict_tools:
return True
caller_asked_for_this_error: Final = anthropic_request_errors_on_thinking_binding_mismatch(
request_body
) and is_anthropic_thinking_binding_mismatch_error(error_text)
@ -219,11 +232,22 @@ class BaseAnthropicMessagesConfig(ABC):
"""
from litellm.llms.anthropic.common_utils import (
is_anthropic_invalid_thinking_block_error,
is_claude_strict_tools_rejection,
strip_thinking_blocks_from_anthropic_messages_request_dict,
tools_without_strict,
warn_claude_strict_tools_dropped,
)
if e.response.status_code == 400 and is_anthropic_invalid_thinking_block_error(e.response.text):
strip_thinking_blocks_from_anthropic_messages_request_dict(request_data)
without_strict: Final = (
tools_without_strict(request_data.get("tools"))
if is_claude_strict_tools_rejection(e.response.status_code, e.response.text)
else None
)
if without_strict is not None:
warn_claude_strict_tools_dropped(without_strict.dropped_count)
request_data["tools"] = list(without_strict.tools) # rebind-ok: the retry loop re-signs this same dict
return request_data
def native_output_fallback_litellm_params(

View file

@ -135,6 +135,18 @@ class BaseTextToSpeechConfig(ABC):
Transform provider response to standard format
"""
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> "HttpxBinaryResponseContent":
return self.transform_text_to_speech_response(
model=model,
raw_response=raw_response,
logging_obj=logging_obj,
)
def get_error_class(self, error_message: str, status_code: int, headers: dict) -> BaseLLMException:
from ..chat.transformation import BaseLLMException

View file

@ -2,11 +2,11 @@ from collections.abc import Callable, Coroutine
from functools import partial
from typing import Final, TypeAlias
from litellm.llms.anthropic.common_utils import is_claude_strict_tools_rejection
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.bedrock.common_utils import (
NativeOutputFallback,
is_bedrock_native_output_rejection,
is_bedrock_strict_tools_rejection,
warn_bedrock_native_output_rejected,
warn_bedrock_strict_tools_dropped,
)
@ -34,7 +34,7 @@ def _native_output_retry(
def _strict_tools_retry(
error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
) -> NativeOutputFallback | None:
if fallback is None or not is_bedrock_strict_tools_rejection(error.status_code, error.message):
if fallback is None or not is_claude_strict_tools_rejection(error.status_code, error.message):
return None
retry: Final = fallback()
if retry is not None:

View file

@ -366,21 +366,10 @@ class NativeOutputFallback:
litellm_params: Mapping[str, object]
_BEDROCK_STRICT_TOOLS_REJECTION: Final = re.compile(
r"Too many strict tools \(\d+\)"
r"|Schemas contains too many (?:optional parameters|parameters with union types)"
r"|[Cc]ompiled grammar (?:is too large|size)"
r"|tools\.\d+\.custom(?:\.strict)?: "
r"|Grammar compilation timed out"
)
_TOOLS_ADAPTER: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_TOOL_SPEC_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def is_bedrock_strict_tools_rejection(status_code: int, error_text: str) -> bool:
return status_code == 400 and _BEDROCK_STRICT_TOOLS_REJECTION.search(error_text) is not None
def warn_bedrock_strict_tools_dropped(model: str) -> None:
verbose_logger.warning(
"Bedrock rejected the strict tool schemas for model=%s, so litellm sent the request again without `strict` "

View file

@ -9,7 +9,7 @@ from collections.abc import Callable, Mapping
from typing import TYPE_CHECKING, Final, get_args, overload
import httpx
from pydantic import TypeAdapter
from pydantic import ConfigDict, TypeAdapter
import litellm
from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL
@ -48,6 +48,8 @@ from .amazon_titan_v2_transformation import AmazonTitanV2Config
from .cohere_transformation import BedrockCohereEmbeddingConfig
from .twelvelabs_marengo_transformation import TwelveLabsMarengoEmbeddingConfig, drop_params_enabled
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -155,7 +157,7 @@ class BedrockEmbedding(BaseAWSLLM):
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
return response.json()
return _JSON_OBJECT.validate_python(response.json())
async def _make_async_call(
self,
@ -189,7 +191,7 @@ class BedrockEmbedding(BaseAWSLLM):
except httpx.TimeoutException:
raise BedrockError(status_code=408, message="Timeout error occurred.")
return response.json()
return _JSON_OBJECT.validate_python(response.json())
def _transform_response(
self,
@ -689,7 +691,7 @@ class BedrockEmbedding(BaseAWSLLM):
# Parse response
if response.status_code == 200:
return response.json()
return _JSON_OBJECT.validate_python(response.json())
else:
raise Exception(f"Failed to get async invoke status: {response.status_code} - {response.text}")

View file

@ -22,6 +22,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
@ -196,7 +197,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
params: Final = cast( # cast-ok: the base signature leaves the params dict untyped
"ResponsesAPIOptionalRequestParams", response_api_optional_request_params
)
hoisted: Final = hoist_additional_tools(input, params.get("tools"))
hoisted: Final = hoist_additional_tools(
input, params.get("tools"), drop_params=drops_unsupported_params(litellm_params.drop_params)
)
normalized_input, rewritten_types = normalize_codex_input_items(hoisted.input)
if rewritten_types:
verbose_logger.warning(

View file

@ -0,0 +1,95 @@
from typing import Final
import litellm
from litellm._logging import verbose_logger
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
from litellm.secret_managers.main import get_secret_str
SUPPORTED_OPENAI_PARAMS: Final = (
"stream",
"stream_options",
"frequency_penalty",
"presence_penalty",
"logit_bias",
"logprobs",
"top_logprobs",
"max_tokens",
"max_completion_tokens",
"n",
"stop",
"temperature",
"top_p",
"seed",
"response_format",
"tools",
"tool_choice",
"parallel_tool_calls",
"user",
"store",
"service_tier",
)
class ClfAiGatewayConfig(OpenAIGPTConfig):
@property
def custom_llm_provider(self) -> str | None:
return "clf_ai_gateway"
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: matches OpenAIGPTConfig's list return
supports_reasoning: Final = litellm.supports_reasoning(
model=model,
custom_llm_provider=self.custom_llm_provider,
)
extra: Final = ("reasoning_effort",) if supports_reasoning else ()
return [*SUPPORTED_OPENAI_PARAMS, *extra]
def map_openai_params(
self,
non_default_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
optional_params: dict[str, object], # mutable-ok: OpenAIConfig.map_openai_params signature
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature
requested: Final = non_default_params.get("reasoning_effort")
requested_mapping: Final = as_str_mapping(requested)
effort: Final = requested_mapping.get("effort") if requested_mapping is not None else requested
declared: Final = declared_reasoning_efforts_for_model(model, "clf_ai_gateway")
drop_enabled: Final = drop_params or litellm.drop_params is True
if requested_mapping is not None and effort is None and not drop_enabled:
raise UnsupportedParamsError(
message=f"clf_ai_gateway needs an effort level in reasoning_effort, got {requested!r}. To drop "
"unsupported openai params from the call, set `litellm.drop_params = True`",
llm_provider="clf_ai_gateway",
model=model,
status_code=400,
)
drops_effort: Final = drop_enabled and effort is not None and declared is not None and effort not in declared
if drops_effort:
verbose_logger.warning(
"Dropping reasoning_effort=%r for model=%r (drop_params=True) because clf_ai_gateway takes only %s for it.",
effort,
model,
declared,
)
others: Final = {param: value for param, value in non_default_params.items() if param != "reasoning_effort"}
normalized: Final = {**others, "reasoning_effort": effort} if effort is not None else others
forwarded: Final = others if drops_effort else normalized
mapped: Final[dict[str, object]] = super().map_openai_params( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # OpenAIGPTConfig types its params and result as bare dicts # mutable-ok: parent returns a dict
non_default_params=forwarded,
optional_params=optional_params,
model=model,
drop_params=drop_params,
)
return mapped
def get_openai_compatible_provider_info(
self, api_base: str | None, api_key: str | None
) -> tuple[str | None, str | None]:
resolved_api_base: Final = (
api_base or get_secret_str("CLF_AI_GATEWAY_API_BASE") or "https://api.clfaigateway.dev/v1"
)
dynamic_api_key: Final = api_key or get_secret_str("CLF_AI_GATEWAY_API_KEY")
return resolved_api_base, dynamic_api_key

View file

@ -1205,6 +1205,17 @@ class AsyncHTTPHandler:
# Mirrors LiteLLMAiohttpTransport._background_close_tasks.
_finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set() # mutable-ok: strong refs for pending closes
@classmethod
async def _close_client_at_finalization(cls, client: httpx.AsyncClient) -> None:
try:
await client.aclose()
except (OSError, RuntimeError) as exc:
verbose_logger.debug("Error closing client at finalization: %s", exc)
finally:
task: Final = asyncio.current_task()
if task is not None:
cls._finalizer_close_tasks.discard(task)
@classmethod
def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
cls._finalizer_close_tasks.discard(task)
@ -1272,8 +1283,8 @@ class AsyncHTTPHandler:
# here is the cross-loop path the transport refuses.
self._dispose_wrapped_aiohttp_session()
return
task: Final = loop.create_task(self._client.aclose())
cls: Final = type(self)
task: Final = loop.create_task(cls._close_client_at_finalization(self._client))
cls._finalizer_close_tasks.add(task)
task.add_done_callback(cls._on_finalizer_close_done)
except Exception:
@ -1973,7 +1984,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
cache = LLMClientCache()
setattr(litellm, "in_memory_llm_clients_cache", cache)
_cached_client: Final = cache.get_cache(_cache_key_name)
_cached_client: Final = cache.get_cache(_cache_key_name, loop_bound=False)
if _cached_client:
return _cached_client
@ -1989,6 +2000,7 @@ def get_httpx_client(params: dict | None = None) -> HTTPHandler:
value=_new_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=True,
loop_bound=False,
)
return _new_client

View file

@ -2467,8 +2467,7 @@ class BaseLLMHTTPHandler:
)
else:
verbose_logger.debug(
"Anthropic /v1/messages: invalid thinking signature; "
"stripping thinking blocks and retrying (attempt %s/%s).",
"Anthropic /v1/messages: retrying after a recoverable HTTP error (attempt %s/%s).",
len(recovered_errors) + 2,
max_attempts,
)
@ -12850,7 +12849,7 @@ class BaseLLMHTTPHandler:
provider_config=text_to_speech_provider_config,
)
speech_response: Final = text_to_speech_provider_config.transform_text_to_speech_response(
speech_response: Final = await text_to_speech_provider_config.async_transform_text_to_speech_response(
model=model,
raw_response=response,
logging_obj=logging_obj,

View file

@ -24,7 +24,8 @@ DASHSCOPE_ANTHROPIC_PATH: Final = "/apps/anthropic"
DASHSCOPE_NATIVE_V1_PATH: Final = "/api/v1"
DASHSCOPE_COMPATIBLE_API_PATH: Final = "/compatible-api/v1"
DASHSCOPE_RERANK_PATH: Final = f"{DASHSCOPE_COMPATIBLE_API_PATH}/reranks"
DASHSCOPE_IMAGE_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_MULTIMODAL_GENERATION_PATH: Final = "/api/v1/services/aigc/multimodal-generation/generation"
DASHSCOPE_IMAGE_PATH: Final = DASHSCOPE_MULTIMODAL_GENERATION_PATH
DASHSCOPE_DEFAULT_ORIGIN: Final = "https://dashscope.aliyuncs.com"
_STANDARD_DASHSCOPE_HOSTS: Final = frozenset(
@ -100,6 +101,21 @@ def is_dashscope_native_endpoint_url(api_base: str) -> bool:
return urlparse(api_base).path.rstrip("/").startswith(f"{DASHSCOPE_NATIVE_V1_PATH}/")
def dashscope_api_root(api_base: str) -> str:
endpoints: Final = resolve_dashscope_endpoints(api_base)
if endpoints is not None:
return endpoints.origin
trimmed: Final = api_base.rstrip("/")
if trimmed.endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH):
return trimmed.removesuffix(DASHSCOPE_CHAT_COMPATIBLE_PATH)
root, separator, _ = trimmed.partition(DASHSCOPE_NATIVE_V1_PATH)
return root if separator else trimmed
def is_aliyun_host(host: str | None) -> bool:
return host is not None and (host == "aliyuncs.com" or host.endswith(".aliyuncs.com"))
def resolve_dashscope_openai_api_base(
api_base: str | None,
env_var: str = "DASHSCOPE_API_BASE",
@ -140,9 +156,7 @@ def _chat_shaped_aliyun_origin(api_base: str | None) -> str | None:
if api_base is None:
return None
parsed: Final = urlparse(api_base)
host: Final = parsed.hostname or ""
on_aliyun_host: Final = host == "aliyuncs.com" or host.endswith(".aliyuncs.com")
if not on_aliyun_host or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
if not is_aliyun_host(parsed.hostname) or parsed.path.rstrip("/") != DASHSCOPE_CHAT_COMPATIBLE_PATH:
return None
return f"{parsed.scheme}://{parsed.netloc}"

View file

@ -0,0 +1,160 @@
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from typing_extensions import override
import litellm
from litellm.images.utils import image_to_data_url
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_CHAT_COMPATIBLE_PATH,
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
dashscope_api_root,
is_dashscope_native_endpoint_url,
resolve_dashscope_endpoints,
)
from litellm.llms.dashscope.image_generation.transformation import (
DEFAULT_API_BASE,
OPENAI_TO_DASHSCOPE_SIZE,
DashScopeImageGenerationConfig,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.images.main import ImageEditOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ImageResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
MAX_INPUT_IMAGES: Final = 3
DASHSCOPE_IMAGE_EDIT_PARAMS: Final = ("n", "size", "negative_prompt", "prompt_extend", "watermark", "seed")
_IMAGES: Final = TypeAdapter(tuple[object, ...])
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
def _deployment_edit_url(api_base: str) -> str:
derives_from_root: Final = not is_dashscope_native_endpoint_url(api_base) and (
resolve_dashscope_endpoints(api_base) is not None
or api_base.rstrip("/").endswith(DASHSCOPE_CHAT_COMPATIBLE_PATH)
)
return f"{dashscope_api_root(api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if derives_from_root else api_base
class DashScopeImageEditConfig(BaseImageEditConfig):
def __init__(self, generation_config: DashScopeImageGenerationConfig | None = None) -> None:
super().__init__()
self._generation_config: Final = generation_config or DashScopeImageGenerationConfig()
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
return list(DASHSCOPE_IMAGE_EDIT_PARAMS)
@override
def map_openai_params(
self,
image_edit_optional_params: ImageEditOptionalRequestParams,
model: str,
drop_params: bool,
) -> dict[str, object]: # mutable-ok: mirrors override contract
return {
key: _dashscope_size(value) if key == "size" else value
for key, value in image_edit_optional_params.items()
if key in DASHSCOPE_IMAGE_EDIT_PARAMS
}
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
litellm_params: dict[str, object] | None = None, # mutable-ok: mirrors override contract
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
return self._generation_config.validate_environment(
headers=dict(headers),
model=model,
messages=[],
optional_params={},
litellm_params=dict(litellm_params or {}),
api_key=api_key,
api_base=api_base,
)
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
if api_base is not None:
return _deployment_edit_url(api_base)
env_api_base: Final = get_secret_str("DASHSCOPE_API_BASE")
derived: Final = (
f"{dashscope_api_root(env_api_base)}{DASHSCOPE_MULTIMODAL_GENERATION_PATH}" if env_api_base else None
)
return get_secret_str("DASHSCOPE_API_BASE_IMAGE") or derived or DEFAULT_API_BASE
@override
def use_multipart_form_data(self) -> bool:
return False
@override
def transform_image_edit_request(
self,
model: str,
prompt: str | None,
image: object,
image_edit_optional_request_params: Mapping[str, object],
litellm_params: GenericLiteLLMParams,
headers: Mapping[str, str],
) -> tuple[dict[str, object], RequestFiles]: # mutable-ok: mirrors override contract
candidates: Final = _IMAGES.validate_python(image) if isinstance(image, list) else (image,)
images: Final = tuple(candidate for candidate in candidates if candidate is not None)
if len(images) > MAX_INPUT_IMAGES:
raise litellm.BadRequestError(
message=f"DashScope image edit accepts at most {MAX_INPUT_IMAGES} input images, got {len(images)}",
model=model,
llm_provider="dashscope",
)
prompt_content: Final = ({"text": prompt},) if prompt is not None else ()
return {
"model": model,
"input": {
"messages": [
{
"role": "user",
"content": [*({"image": image_to_data_url(item)} for item in images), *prompt_content],
}
]
},
"parameters": dict(image_edit_optional_request_params),
}, ()
@override
def transform_image_edit_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> ImageResponse:
return self._generation_config.transform_image_generation_response(
model=model,
raw_response=raw_response,
model_response=ImageResponse(),
logging_obj=logging_obj,
request_data={},
optional_params={},
litellm_params={},
encoding=None,
)

View file

@ -0,0 +1,19 @@
from typing import Final
import litellm
from litellm.litellm_core_utils.llm_cost_calc.utils import resolve_image_model_info
from litellm.types.utils import ImageResponse, ModelInfo
def cost_calculator(model: str, image_response: ImageResponse, model_info: ModelInfo | None = None) -> float:
prices: Final = resolve_image_model_info(
model=model, custom_llm_provider=litellm.LlmProviders.DASHSCOPE.value, model_info=model_info
)
output_cost_per_image: Final = prices.get("output_cost_per_image")
if output_cost_per_image is None:
raise ValueError(f"No output_cost_per_image found for dashscope/{model}")
input_images: Final = image_response.hidden_params.get("billable_input_image_count")
input_image_cost: Final = (prices.get("input_cost_per_image") or 0.0) * (
input_images if isinstance(input_images, int) else 0
)
return output_cost_per_image * len(image_response.data or ()) + input_image_cost

View file

@ -23,7 +23,7 @@ Response format:
}
"""
from collections.abc import Iterable, Mapping
from collections.abc import Iterable, Iterator, Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -61,7 +61,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"
# Maps OpenAI size strings (WxH) to DashScope size strings (W*H)
OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
OPENAI_TO_DASHSCOPE_SIZE: Final[Mapping[str, str]] = {
"256x256": "256*256",
"512x512": "512*512",
"1024x1024": "1024*1024",
@ -71,6 +71,24 @@ OPENAI_TO_DASHSCOPE_SIZE: Final[dict] = {
}
def _message_image_urls(message: Mapping[str, object]) -> Iterator[str]:
contents: Final = _JSON_OBJECTS.validate_python(message.get("content", []))
return (url for url in (item.get("image") for item in contents) if isinstance(url, str) and url)
def _image_urls(response_object: Mapping[str, object]) -> Iterator[str]:
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
yield from _message_image_urls(_JSON_OBJECT.validate_python(choice.get("message", {})))
def _dashscope_size(size: object) -> object:
if not isinstance(size, str):
return size
return OPENAI_TO_DASHSCOPE_SIZE.get(size, size.replace("x", "*"))
class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
"""
Configuration for DashScope image generation (qwen-image-2.0, qwen-image-2.0-pro,
@ -82,24 +100,16 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def map_openai_params(
self,
non_default_params: dict,
optional_params: dict,
non_default_params: dict[str, object],
optional_params: dict[str, object],
model: str,
drop_params: bool,
) -> dict:
) -> dict[str, object]:
supported_params: Final = self.get_supported_openai_params(model)
mapped: Final[dict] = {}
for k, v in non_default_params.items():
if k in optional_params:
continue
if k not in supported_params:
continue
if k == "size":
# Convert "WxH" → "W*H"
mapped["size"] = OPENAI_TO_DASHSCOPE_SIZE.get(v, v.replace("x", "*"))
else:
mapped[k] = v
return mapped
requested: Final = {
k: v for k, v in non_default_params.items() if k in supported_params and k not in optional_params
}
return {k: _dashscope_size(v) if k == "size" else v for k, v in requested.items()}
def _resolve_api_key(self, api_key: str | None) -> str:
resolved_api_key: Final = api_key or get_secret_str("DASHSCOPE_API_KEY")
@ -121,8 +131,8 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
api_base: str | None,
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
stream: bool | None = None,
) -> str:
image_env_api_base: Final = self._image_api_base_env()
@ -140,14 +150,14 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
def validate_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
optional_params: dict[str, object],
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
) -> dict[str, str]:
headers["Authorization"] = f"Bearer {self._resolve_api_key(api_key)}"
headers["Content-Type"] = "application/json"
return headers
@ -186,9 +196,9 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
raw_response: httpx.Response,
model_response: ImageResponse,
logging_obj: LiteLLMLoggingObj,
request_data: dict,
optional_params: dict,
litellm_params: dict,
request_data: dict[str, object],
optional_params: dict[str, object],
litellm_params: dict[str, object],
encoding: "Tokenizer | None",
api_key: str | None = None,
json_mode: bool | None = None,
@ -225,17 +235,21 @@ class DashScopeImageGenerationConfig(BaseImageGenerationConfig):
headers=raw_response.headers,
)
if not model_response.data:
model_response.data = []
output: Final = _JSON_OBJECT.validate_python(response_object.get("output", {}))
choices: Final = _JSON_OBJECTS.validate_python(output.get("choices", []))
for choice in choices:
message = _JSON_OBJECT.validate_python(choice.get("message", {}))
content_list = _JSON_OBJECTS.validate_python(message.get("content", []))
for content_item in content_list:
image_url = content_item.get("image")
if image_url:
model_response.data.append(ImageObject(url=image_url))
return model_response
images: Final = [*(model_response.data or ()), *(ImageObject(url=url) for url in _image_urls(response_object))]
usage: Final = response_object.get("usage")
usage_fields: Final[Mapping[str, object]] = usage if isinstance(usage, Mapping) else {}
width: Final = usage_fields.get("output_width", usage_fields.get("width"))
height: Final = usage_fields.get("output_height", usage_fields.get("height"))
image_response: Final = model_response.model_copy(
update={
"data": images,
**({"size": f"{width}x{height}"} if isinstance(width, int) and isinstance(height, int) else {}),
}
)
input_image_count: Final = usage_fields.get("input_image_count")
if isinstance(input_image_count, int) and input_image_count > 0:
image_response.hidden_params = {
**model_response.hidden_params,
"billable_input_image_count": input_image_count,
}
return image_response

View file

@ -0,0 +1,297 @@
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from itertools import chain
from typing import TYPE_CHECKING, Final
from urllib.parse import urlparse
import httpx
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import override
import litellm
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.url_utils import SSRFError, async_safe_get, safe_get
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
get_httpx_client, # pyright: ignore[reportUnknownVariableType] # factory stub leaves params untyped
)
from litellm.llms.dashscope.common_utils import (
DASHSCOPE_MULTIMODAL_GENERATION_PATH,
DashScopeError,
dashscope_api_root,
is_aliyun_host,
missing_dashscope_family_key_message,
resolve_dashscope_family_api_key,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import HttpxBinaryResponseContent
from litellm.types.utils import Usage
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
DASHSCOPE_TTS_DEFAULT_API_BASE: Final = "https://dashscope-intl.aliyuncs.com"
_SPEECH_SYNTHESIZER_MODEL_PREFIX: Final = "qwen-audio-"
_TIMEOUT_PHASES: Final = TypeAdapter(dict[str, float | None])
@dataclass(frozen=True, slots=True)
class _TextToSpeechFamily:
path: str
response_formats: frozenset[str]
sends_format: bool
takes_instructions: bool
MULTIMODAL_GENERATION_FAMILY: Final = _TextToSpeechFamily(
path=DASHSCOPE_MULTIMODAL_GENERATION_PATH,
response_formats=frozenset({"wav"}),
sends_format=False,
takes_instructions=False,
)
SPEECH_SYNTHESIZER_FAMILY: Final = _TextToSpeechFamily(
path="/api/v1/services/audio/tts/SpeechSynthesizer",
response_formats=frozenset({"mp3", "wav", "opus", "pcm"}),
sends_format=True,
takes_instructions=True,
)
class _Audio(BaseModel):
model_config = ConfigDict(frozen=True)
url: str
class _Output(BaseModel):
model_config = ConfigDict(frozen=True)
audio: _Audio
class _Usage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: int | None = None
output_tokens: int | None = None
characters: int | None = None
class _SpeechResponse(BaseModel):
model_config = ConfigDict(frozen=True)
output: _Output | None = None
usage: _Usage | None = None
code: str | None = None
message: str | None = None
def text_to_speech_family(model: str) -> _TextToSpeechFamily:
if model.startswith(_SPEECH_SYNTHESIZER_MODEL_PREFIX):
return SPEECH_SYNTHESIZER_FAMILY
return MULTIMODAL_GENERATION_FAMILY
def _mapped_field(family: _TextToSpeechFamily, key: str, value: object) -> Mapping[str, object] | None:
match key:
case "response_format" if value in family.response_formats:
return {"format": value} if family.sends_format else {}
case "instructions" if family.takes_instructions:
return {"instruction": value}
case _:
return None
def _unsupported(model: str, unsupported: tuple[str, ...]) -> UnsupportedParamsError:
return UnsupportedParamsError(
message=f"DashScope {model} does not support {', '.join(unsupported)}, set drop_params=True to ignore it",
model=model,
llm_provider="dashscope",
)
def _speech_response(raw_response: httpx.Response) -> _SpeechResponse:
try:
return _SpeechResponse.model_validate_json(raw_response.content)
except ValidationError as e:
raise DashScopeError(
status_code=502,
message=f"Unexpected DashScope text-to-speech response: {e}",
headers=raw_response.headers,
) from e
def _https_on_aliyun(url: str) -> str:
parsed: Final = urlparse(url)
return (
parsed._replace(scheme="https").geturl() if parsed.scheme == "http" and is_aliyun_host(parsed.hostname) else url
)
def _audio_url(raw_response: httpx.Response) -> tuple[str, _Usage | None]:
parsed: Final = _speech_response(raw_response)
if parsed.output is None:
raise DashScopeError(
status_code=max(raw_response.status_code, 400),
message=parsed.message or parsed.code or raw_response.text,
headers=raw_response.headers,
)
return _https_on_aliyun(parsed.output.audio.url), parsed.usage
def _request_timeout(raw_response: httpx.Response) -> httpx.Timeout | None:
extension: Final = raw_response.request.extensions.get("timeout")
if extension is None:
return None
phases: Final = _TIMEOUT_PHASES.validate_python(extension)
return httpx.Timeout(
connect=phases.get("connect"), read=phases.get("read"), write=phases.get("write"), pool=phases.get("pool")
)
def _token_usage(usage: _Usage | None) -> Usage | None:
if usage is None or usage.input_tokens is None or usage.output_tokens is None:
return None
return Usage(
prompt_tokens=usage.input_tokens,
completion_tokens=usage.output_tokens,
total_tokens=usage.input_tokens + usage.output_tokens,
)
def _refused_audio_url(error: SSRFError) -> DashScopeError:
return DashScopeError(status_code=502, message=f"DashScope returned an audio URL that was refused: {error}")
def _binary_audio(audio_response: httpx.Response, usage: _Usage | None) -> HttpxBinaryResponseContent:
if audio_response.status_code != 200:
raise DashScopeError(
status_code=502,
message=f"DashScope audio download failed with HTTP {audio_response.status_code}",
headers=audio_response.headers,
)
binary: Final = HttpxBinaryResponseContent(audio_response)
binary.usage = _token_usage(usage)
if usage is not None and usage.characters is not None:
binary.hidden_params = {**binary.hidden_params, "billed_characters": usage.characters}
return binary
def _shared_async_client() -> AsyncHTTPHandler:
return get_async_httpx_client(llm_provider=litellm.LlmProviders.DASHSCOPE)
class DashScopeTextToSpeechConfig(BaseTextToSpeechConfig):
def __init__(
self,
sync_client: Callable[[], HTTPHandler] = get_httpx_client,
async_client: Callable[[], AsyncHTTPHandler] = _shared_async_client,
) -> None:
super().__init__()
self._sync_client: Final = sync_client
self._async_client: Final = async_client
@override
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
return ["voice", "response_format", *(("instructions",) if family.takes_instructions else ())]
@override
def map_openai_params(
self,
model: str,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
voice: str | dict[str, object] | None = None, # mutable-ok: mirrors override contract
drop_params: bool = False,
kwargs: Mapping[str, object] | None = None,
) -> tuple[str | None, dict[str, object]]: # mutable-ok: mirrors override contract
family: Final = text_to_speech_family(model)
fields: Final = tuple((key, value, _mapped_field(family, key, value)) for key, value in optional_params.items())
unsupported: Final = (
*(f"{key}={value!r}" for key, value, field in fields if field is None),
*(("voice as an object",) if isinstance(voice, dict) else ()),
)
if unsupported and not (drop_params or drops_unsupported_params((kwargs or {}).get("drop_params"))):
raise _unsupported(model, unsupported)
supported: Final = tuple(field for _, _, field in fields if field is not None)
return voice if isinstance(voice, str) else None, dict(chain.from_iterable(f.items() for f in supported))
@override
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: mirrors override contract
model: str,
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: mirrors override contract
resolved_api_key: Final = resolve_dashscope_family_api_key("dashscope", api_key)
if resolved_api_key is None:
raise ValueError(missing_dashscope_family_key_message("dashscope"))
return {**headers, "Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"}
@override
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
) -> str:
configured: Final = api_base or get_secret_str("DASHSCOPE_API_BASE")
root: Final = dashscope_api_root(configured) if configured else DASHSCOPE_TTS_DEFAULT_API_BASE
return f"{root}{text_to_speech_family(model).path}"
@override
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: str | None,
optional_params: dict[str, object], # mutable-ok: mirrors override contract
litellm_params: dict[str, object], # mutable-ok: mirrors override contract
headers: dict[str, str], # mutable-ok: mirrors override contract
) -> TextToSpeechRequestData:
voice_field: Final = {"voice": voice} if voice is not None else {}
return {"dict_body": {"model": model, "input": {"text": input, **voice_field, **optional_params}}}
@override
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = safe_get(self._sync_client(), audio_url, timeout=_request_timeout(raw_response))
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
async def async_transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> HttpxBinaryResponseContent:
audio_url, usage = _audio_url(raw_response)
try:
audio_response: Final = await async_safe_get(
self._async_client(), audio_url, timeout=_request_timeout(raw_response)
)
except SSRFError as e:
raise _refused_audio_url(e) from e
return _binary_audio(audio_response, usage)
@override
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str] | httpx.Headers, # mutable-ok: mirrors override contract
) -> BaseLLMException:
return DashScopeError(status_code=status_code, message=error_message, headers=httpx.Headers(headers))

View file

@ -9,6 +9,7 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.litellm_core_utils.core_helpers import set_response_cost_in_hidden_params
@ -21,6 +22,8 @@ from litellm.utils import convert_to_model_response_object
from ..common_utils import EdenAIException, authorized_headers, endpoint_url, reported_cost
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
def _form_fields(model: str, optional_params: Mapping[str, object]) -> dict[str, object]: # mutable-ok: httpx form data
"""LiteLLM parks non-OpenAI params, `model` included, under `extra_body` for the OpenAI SDK; a
@ -75,7 +78,7 @@ class EdenAIAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig):
def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
if "application/json" not in raw_response.headers.get("content-type", ""):
return TranscriptionResponse(text=raw_response.text)
body: Final = raw_response.json()
body: Final = _JSON_OBJECT.validate_python(raw_response.json())
response: Final[TranscriptionResponse] = convert_to_model_response_object(
response_object=body, model_response_object=TranscriptionResponse(), response_type="audio_transcription"
)

View file

View file

@ -0,0 +1,34 @@
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
EU_JEV_LATEST_MODEL: Final = "eu-jeff-latest"
_EU_JEV_CURRENT_RELEASE_NAMES: Final = frozenset((EU_JEV_LATEST_MODEL, "jeff-latest", "jeff-1.0.0"))
_EU_JEV_PREFIX: Final = "eu_jev/"
def eu_jev_registry_model(requested_model: str | None) -> str:
release: Final = requested_model.removeprefix(_EU_JEV_PREFIX) if requested_model is not None else None
if release is None or release in _EU_JEV_CURRENT_RELEASE_NAMES:
return EU_JEV_LATEST_MODEL
return release
def eu_jev_scoped_model(decision_body: Mapping[str, object]) -> str:
requested_model: Final = decision_body.get("model")
return f"{_EU_JEV_PREFIX}{eu_jev_registry_model(requested_model if isinstance(requested_model, str) else None)}"
def eu_jev_upstream_body(hook_body: Mapping[str, object], client_body: Mapping[str, object]) -> Mapping[str, object]:
hook_model: Final = hook_body.get("model")
without_model: Final = {key: value for key, value in hook_body.items() if key != "model"}
if hook_model == eu_jev_scoped_model(client_body):
client_model: Final = client_body.get("model")
if client_model is None:
return MappingProxyType(without_model)
if isinstance(client_model, str):
return MappingProxyType({**without_model, "model": client_model.removeprefix(_EU_JEV_PREFIX)})
return MappingProxyType({**without_model, "model": client_model})
if isinstance(hook_model, str):
return MappingProxyType({**without_model, "model": hook_model.removeprefix(_EU_JEV_PREFIX)})
return hook_body

View file

@ -0,0 +1,9 @@
from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
class EuJevDecisionsConfig(BaseDecisionsConfig):
api_key_env = ("EU_JEV_API_KEY",)
api_base_env = ("EU_JEV_API_BASE",)
def get_default_api_base(self) -> str | None:
return "https://eu-jev.com"

View file

@ -5,11 +5,30 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod
This config enables pass-through behavior to the proxy's /v1/responses endpoint.
"""
from typing import Final
import httpx
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import LlmProviders
_ERROR_BODY: Final = TypeAdapter(dict[str, object])
def _openai_sdk_error_body(
error_message: str,
) -> dict[str, object] | None: # mutable-ok: BaseLLMException.body is a dict
try:
decoded: Final = _ERROR_BODY.validate_json(error_message)
except ValidationError:
return None
nested: Final = decoded.get("error")
return _ERROR_BODY.validate_python(nested) if isinstance(nested, dict) else decoded
class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
"""
@ -43,6 +62,19 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig):
return join_api_base(api_base, "responses")
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, object] | httpx.Headers, # mutable-ok: inherited contract
) -> BaseLLMException:
return BaseLLMException(
status_code=status_code,
message=error_message,
headers=headers,
body=_openai_sdk_error_body(error_message),
)
def supports_native_websocket(self) -> bool:
"""LiteLLM Proxy does not support native WebSocket for Responses API"""
return False

View file

@ -687,11 +687,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
responses_so_far: Sequence["ModelResponseStream"],
litellm_logging_obj: "LiteLLMLoggingObj | None",
) -> "ModelResponse":
"""``stream_chunk_builder`` folds every choice of a stream into one index-0
choice, so the stream is rebuilt one choice index at a time (every chunk
kept, its choices narrowed to that index, so usage-only chunks still
count) and the rebuilt choices are stitched into one response, each
carrying the index the stream gave it."""
choice_indices: Final = tuple(
sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
)

View file

@ -275,6 +275,10 @@ def build_output_token_limit_response(
return e.response.headers, (_aiter_once(chunk) if is_async else _iter_once(chunk))
def _is_loop_bound(client_initialization_params: Mapping[str, object]) -> bool:
return client_initialization_params.get("is_async") is not False
class BaseOpenAILLM:
"""
Base class for OpenAI LLMs for getting their httpx clients and SSL verification settings
@ -289,7 +293,9 @@ class BaseOpenAILLM:
client_initialization_params=client_initialization_params,
client_type=client_type,
)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(_cache_key)
_cached_client: Final = litellm.in_memory_llm_clients_cache.get_cache(
_cache_key, loop_bound=_is_loop_bound(client_initialization_params)
)
return _cached_client
@staticmethod
@ -330,6 +336,7 @@ class BaseOpenAILLM:
value=openai_client,
ttl=_DEFAULT_TTL_FOR_HTTPX_CLIENTS,
litellm_owned_client=litellm_owned_client,
loop_bound=_is_loop_bound(client_initialization_params),
)
@staticmethod

View file

@ -1,6 +1,7 @@
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
@ -9,6 +10,8 @@ from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
from litellm.types.utils import ImageResponse
from litellm.utils import convert_to_model_response_object
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
if TYPE_CHECKING:
from litellm.litellm_core_utils.logging import Logging as LiteLLMLoggingObj
@ -66,7 +69,7 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
api_key: str | None = None,
json_mode: bool | None = None,
) -> ImageResponse:
response: Final = raw_response.json()
response: Final = _JSON_OBJECT.validate_python(raw_response.json())
stringified_response: Final = response
## LOGGING

View file

@ -15,7 +15,9 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.tool_fields import DropPolicy
from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
from litellm.llms.anthropic.messages.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
from litellm.llms.anthropic.messages.responses_adapters.handler import count_view_messages
from litellm.llms.anthropic.messages.responses_adapters.transformation import (
LiteLLMAnthropicToResponsesAPIAdapter,
)
@ -117,6 +119,32 @@ def _with_deployment_reasoning(
return without_reasoning if resolved is None else {**without_reasoning, "reasoning": resolved}
async def _sent_messages(
model: str,
messages: Sequence[Mapping[str, object]],
tools: Sequence[Mapping[str, object]] | None,
system: object,
messages_format: TokenCountMessagesFormat,
context_management: CountContextManagement | None,
) -> Sequence[Mapping[str, object]] | AnthropicContextManagementError:
match messages_format:
case "openai":
return messages
case "anthropic":
try:
return await count_view_messages(
model=model,
messages=messages,
tools=tools,
system=system,
context_management=context_management,
)
except AnthropicContextManagementError as error:
return error
case _:
assert_never(messages_format)
class OpenAITokenCounter(BaseTokenCounter):
"""Token counter implementation for OpenAI provider using the Responses API."""
@ -197,13 +225,28 @@ class OpenAITokenCounter(BaseTokenCounter):
rejected_before_provider_call=True,
)
sent_messages: Final = await _sent_messages(
model_to_use, messages, tools, system, messages_format, context_management
)
if isinstance(sent_messages, AnthropicContextManagementError):
return TokenCountResponse(
total_tokens=0,
request_model=request_model,
model_used=model_to_use,
tokenizer_type="openai_api",
error=True,
error_message=sent_messages.message,
status_code=sent_messages.status_code,
rejected_before_provider_call=True,
)
resolved: Final = _count_credential(deployment, request_model or model_to_use)
if isinstance(resolved, OpenAIError):
return _count_error(model_to_use, request_model, resolved.message, resolved.status_code)
input_items, instructions, counted_tools, text = self._responses_request_parts(
model_to_use=model_to_use,
messages=messages,
messages=sent_messages,
tools=tools,
system=system,
messages_format=messages_format,

View file

@ -11,6 +11,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.litellm_logging import (
get_standard_logging_object_payload, # pyright: ignore[reportUnknownVariableType] # legacy helper has an untyped signature
)
from litellm.llms.eu_jev.common_utils import EU_JEV_LATEST_MODEL, eu_jev_registry_model
from litellm.llms.laya.common_utils import laya_response_model
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
@ -82,13 +83,21 @@ class TypeSafePassthroughLoggingHandler:
response_model: Final = (
laya_response_model(response_body, request_model) if custom_llm_provider == "laya" else response.model
)
logged_model: Final = response_model or request_model or "unknown"
registry_request_model: Final = (
eu_jev_registry_model(request_model) if custom_llm_provider == "eu_jev" else request_model
)
logged_model: Final = response_model or registry_request_model or "unknown"
model_name: Final = f"{custom_llm_provider}/{logged_model}"
usage: Final = response.usage or _TypeSafeUsage()
input_tokens: Final = usage.input_tokens
output_tokens: Final = usage.output_tokens
candidate_models: Final = (
(response_model, registry_request_model, EU_JEV_LATEST_MODEL)
if custom_llm_provider == "eu_jev"
else (response_model, registry_request_model)
)
candidate_model_keys: Final = tuple(
f"{custom_llm_provider}/{model}" for model in (response_model, request_model) if model is not None
f"{custom_llm_provider}/{model}" for model in candidate_models if model is not None
)
pricing: Final = _pricing_for(candidate_model_keys)
response_cost: Final = (

View file

@ -15,8 +15,10 @@ from litellm.litellm_core_utils.url_utils import (
)
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
MaskedHTTPStatusError,
get_async_httpx_client,
get_httpx_client,
mask_sensitive_info,
)
from litellm.llms.vertex_ai.common_utils import VertexAIError, get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
@ -81,6 +83,15 @@ def _vertex_batch_list_payload(response: _VertexBatchListJsonSource) -> dict[str
return response.json()
def _vertex_batch_create_error(error: httpx.HTTPStatusError) -> MaskedHTTPStatusError:
body: Final = mask_sensitive_info(error.response.text)
return MaskedHTTPStatusError(
httpx.HTTPStatusError(f"{error}\n{body}", request=error.request, response=error.response),
message=body,
text=body,
)
class VertexAIBatchPrediction(VertexLLM):
def __init__(self, gcs_bucket_name: str, *args, **kwargs):
super().__init__(*args, **kwargs)
@ -166,11 +177,19 @@ class VertexAIBatchPrediction(VertexLLM):
headers=headers,
)
response: Final = sync_handler.post(
url=api_base,
headers=headers,
data=json.dumps(vertex_batch_request),
)
try:
response: Final = sync_handler.post(
url=api_base,
headers=headers,
data=json.dumps(vertex_batch_request),
)
except httpx.HTTPStatusError as e:
litellm.verbose_logger.error(
"Vertex AI batch create failed: status=%s, body=%s",
e.response.status_code,
e.response.text[:1000],
)
raise _vertex_batch_create_error(e) from e
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
_json_response: Final = payload_view["payload"]
@ -266,13 +285,12 @@ class VertexAIBatchPrediction(VertexLLM):
data=json.dumps(vertex_batch_request),
)
except httpx.HTTPStatusError as e:
error_body: Final = e.response.text
litellm.verbose_logger.error(
"Vertex AI batch create failed: status=%s, body=%s",
e.response.status_code,
error_body[:1000],
e.response.text[:1000],
)
raise
raise _vertex_batch_create_error(e) from e
payload_view: Final[_VertexBatchPayloadView] = {"payload": response.json()}
_json_response: Final = payload_view["payload"]

View file

@ -9,14 +9,41 @@ from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.llms.vertex_ai.common_utils import (
VertexAIError,
VertexAIModelRoute,
convert_vertex_datetime_to_openai_datetime,
get_vertex_ai_model_route,
)
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
from litellm.types.llms.openai import BatchJobStatus, CreateBatchRequest
from litellm.types.llms.vertex_ai import *
from litellm.types.llms.vertex_ai import GenerateContentResponseBody
from litellm.types.llms.vertex_ai import GenerateContentResponseBody, VertexPartnerProvider
from litellm.types.utils import LiteLLMBatch, ModelInfo, Usage
_NATIVE_VERTEX_RESPONSE: Final = TypeAdapter(GenerateContentResponseBody)
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
VERTEX_ANTHROPIC_VERSION: Final = "vertex-2023-10-16"
_ANTHROPIC_PUBLISHER_MODEL_PATH: Final = "publishers/anthropic/models/"
def is_vertex_anthropic_batch_model(model: str) -> bool:
"""
Whether a Vertex batch model (bare, `vertex_ai/`-prefixed or a publisher path) is a Claude model, decided the
way Vertex chat routes Claude. Vertex batches Claude only as `publishers/anthropic/models/<model>` with rows in
Anthropic's Messages shape
"""
bare_model: Final = model.rsplit("/", 1)[-1]
return _ANTHROPIC_PUBLISHER_MODEL_PATH in model or (
get_vertex_ai_model_route(bare_model) is VertexAIModelRoute.PARTNER_MODELS
and VertexPartnerProvider.claude.value in bare_model
)
def as_json_object(value: object) -> Mapping[str, object] | None:
return _JSON_OBJECT.validate_python(value) if isinstance(value, dict) else None
def is_vertex_anthropic_batch_output_response(response_body: Mapping[str, object]) -> bool:
return response_body.get("type") == "message" and isinstance(response_body.get("usage"), dict)
def _int_field(mapping: Mapping[str, object], key: str) -> int:
@ -43,8 +70,22 @@ def is_vertex_embedding_batch_output_response(response_body: Mapping[str, object
return isinstance(response_body.get("embedding"), dict)
def is_vertex_anthropic_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
"""
Whether a Vertex batch output row came from a Claude (Anthropic Messages) request. Vertex echoes only the
request's top-level `custom_id` on these rows, with an Anthropic `message` response on success and an empty
response plus a `status` on failure, so a Claude input file read back is never mistaken for output
"""
if not ("custom_id" in vertex_output_row and "processed_time" in vertex_output_row):
return False
response: Final = as_json_object(vertex_output_row.get("response"))
return (response is not None and is_vertex_anthropic_batch_output_response(response)) or bool(
vertex_output_row.get("status")
)
def is_native_vertex_batch_output_row(row: Mapping[str, object]) -> bool:
return isinstance(row.get("request"), dict)
return isinstance(row.get("request"), dict) or is_vertex_anthropic_batch_output_row(row)
class NativeVertexBatchCostCalculator(Protocol):
@ -72,6 +113,9 @@ def _native_vertex_row_usage(
response_body: Mapping[str, object],
calculate_usage: Callable[[GenerateContentResponseBody], Usage],
) -> Usage | None:
anthropic_usage: Final = as_json_object(response_body.get("usage"))
if is_vertex_anthropic_batch_output_response(response_body) and anthropic_usage is not None:
return VertexAIAnthropicConfig().calculate_usage(usage_object=anthropic_usage, reasoning_content=None)
if "usageMetadata" not in response_body:
if not is_vertex_embedding_batch_output_response(response_body):
return None
@ -97,21 +141,24 @@ def native_vertex_batch_row_stats(
"""
Usage and cost of one native Vertex predictions.jsonl row, a
`{"request": ..., "response": {"candidates": [...], "usageMetadata": {...}, "modelVersion": ...}}`
generateContent object or a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`).
`model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion`
does, else the wildcard name so explicit deployment prices still apply; a row without a response, a
generateContent row without `response.usageMetadata`, and a row whose response fails validation are
generateContent object, a `{"request": ..., "response": {"embedding": {...}, "usageMetadata": {...}}}`
embedding object (an embedding row without `usageMetadata` is billed from its documented `tokenCount`) or a
`{"custom_id": ..., "request": ..., "response": {"type": "message", "model": ..., "usage": {...}}}` Claude object.
`model_name` (the deployment model) prices the row unless it is a wildcard, else its own `modelVersion` (Claude
rows: `model`) does, else the wildcard name so explicit deployment prices still apply; a row without a response,
a generateContent row without `response.usageMetadata`, and a row whose response fails validation are
None (failed).
"""
response_body: Final = row.get("response")
if not isinstance(response_body, dict):
response_body: Final = as_json_object(row.get("response"))
if response_body is None:
return None
usage: Final = _native_vertex_row_usage(response_body, calculate_usage)
if usage is None:
return None
total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
model_version: Final = response_body.get("modelVersion")
model_version: Final = response_body.get(
"model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
)
deployment_model: Final = model_name if model_name and "*" not in model_name else None
model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
if model is None:

View file

@ -21,6 +21,7 @@ from typing_extensions import ReadOnly, Required
import litellm
from litellm._uuid import uuid
from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, RESPONSE_FORMAT_TOOL_NAME
from litellm.files.types import FileContentStreamingResult
from litellm.files.utils import FilesAPIUtils
from litellm.litellm_core_utils.cloud_storage_security import (
@ -32,13 +33,14 @@ from litellm.litellm_core_utils.cloud_storage_security import (
split_configured_cloud_bucket_name,
validate_managed_cloud_file_id,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.prompt_templates.common_utils import (
extract_file_data,
extract_file_metadata,
)
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.anthropic.chat.transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@ -47,7 +49,14 @@ from litellm.llms.base_llm.files.transformation import (
BaseFileUploadStream,
LiteLLMLoggingObj,
)
from litellm.llms.vertex_ai.batches.transformation import vertex_embedding_prompt_token_count
from litellm.llms.vertex_ai.batches.transformation import (
VERTEX_ANTHROPIC_VERSION,
as_json_object,
is_vertex_anthropic_batch_model,
is_vertex_anthropic_batch_output_response,
is_vertex_anthropic_batch_output_row,
vertex_embedding_prompt_token_count,
)
from litellm.llms.vertex_ai.common_utils import (
convert_vertex_datetime_to_openai_datetime,
get_vertex_ai_fine_tuned_endpoint_id,
@ -59,6 +68,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import (
transform_openai_input_gemini_embed_content,
)
from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import VertexAIAnthropicConfig
from litellm.types.files import StreamingMediaUploadConfig
from litellm.types.llms.openai import (
AllMessageValues,
@ -79,6 +89,7 @@ from litellm.types.utils import (
ModelResponse,
Usage,
)
from litellm.utils import get_non_default_completion_params, get_optional_params
from ..common_utils import VertexAIError
from ..vertex_llm_base import VertexBase
@ -87,6 +98,7 @@ _GCP_LABEL_VALUE_MAX_LEN: Final = 63
_CUSTOM_ID_RAW_LABEL_PREFIX: Final = "b32_"
_VERTEX_BATCH_KEY_FIELD: Final = "key"
_MANAGED_GCS_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/([^/?]+)")
_PUBLISHER_MODEL_PATH_PATTERN: Final = re.compile(r"publishers/[^/]+/models/")
_EMBED_REQUEST_FIELD_BY_GEMINI_PARAM: Final = (
("outputDimensionality", "output_dimensionality"),
("taskType", "task_type"),
@ -225,10 +237,14 @@ def _get_litellm_batch_custom_id(vertex_output_row: Mapping[str, object]) -> str
"""
Resolve the OpenAI `custom_id` for a Vertex batch output row.
Embedding rows carry it in the top-level `key` field that Vertex echoes back;
`generateContent` rows have no such field, so it is smuggled through request
labels instead (see `_set_litellm_batch_custom_id_labels`).
Claude rows carry it in their own top-level `custom_id` field and embedding rows in
the top-level `key` field that Vertex echoes back; `generateContent` rows have no
such field, so it is smuggled through request labels instead (see
`_set_litellm_batch_custom_id_labels`).
"""
anthropic_custom_id: Final = vertex_output_row.get("custom_id")
if anthropic_custom_id is not None:
return str(anthropic_custom_id)
key = vertex_output_row.get(_VERTEX_BATCH_KEY_FIELD)
if key is not None:
return unquote(str(key))
@ -289,6 +305,12 @@ def _is_vertex_generate_content_batch_output_row(vertex_output_row: Mapping[str,
)
def _is_vertex_chat_batch_output_row(vertex_output_row: Mapping[str, object]) -> bool:
return _is_vertex_generate_content_batch_output_row(vertex_output_row) or is_vertex_anthropic_batch_output_row(
vertex_output_row
)
def _try_parse_vertex_batch_output_row(line: bytes) -> _VertexBatchRow | None:
try:
row: Final = _parse_vertex_batch_output_row(line.decode("utf-8"))
@ -410,6 +432,37 @@ def _openai_batch_output_row(
}
def _vertex_anthropic_batch_output_row_to_openai(
custom_id: str,
response: object,
mock_httpx_response: httpx.Response,
) -> _OpenAIBatchOutputRow:
"""
One OpenAI batch output row from a Vertex Claude row's Anthropic `message` response
"""
anthropic_response: Final = as_json_object(response)
if anthropic_response is None or not is_vertex_anthropic_batch_output_response(anthropic_response):
return _openai_batch_output_row(
custom_id=custom_id,
error_code="vertex_ai_error",
error_message="Claude batch row has no message response",
)
try:
model_response: Final = VertexAIAnthropicConfig().transform_parsed_response(
completion_response=dict(anthropic_response),
raw_response=mock_httpx_response,
model_response=ModelResponse(),
json_mode=True,
)
except Exception as e: # noqa: BLE001 # one malformed row must fail alone, not the whole output file
return _openai_batch_output_row(
custom_id=custom_id,
error_code="transformation_error",
error_message=f"Failed to transform response: {e}",
)
return _openai_batch_output_row(custom_id=custom_id, body=model_response.model_dump())
def _split_vertex_batch_key(vertex_output_row: Mapping[str, object]) -> tuple[str, int, int]:
"""
Resolve `(custom_id, index within that custom_id, group size)` for a Vertex batch
@ -684,21 +737,123 @@ def _openai_batch_jsonl_entry_to_vertex_embeddings_rows(
)
_CLAUDE_BATCH_ROW_FIXED_PARAMS: Final = frozenset(
("model", "messages", "custom_llm_provider", "drop_params", "stream", "stream_options")
)
_CLAUDE_BATCH_OPENAI_PARAMS: Final = frozenset(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) - {"metadata"}
_CLAUDE_BATCH_DEPLOYMENT_PARAMS: Final = (
_CLAUDE_BATCH_OPENAI_PARAMS - _CLAUDE_BATCH_ROW_FIXED_PARAMS - {"extra_headers"}
) | {"drop_params"}
def _claude_batch_deployment_params(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
return MappingProxyType(
{key: value for key, value in litellm_params.items() if key in _CLAUDE_BATCH_DEPLOYMENT_PARAMS}
)
def _names_the_response_format_tool(tool: object) -> bool:
match tool:
case {"function": {"name": str(name)}}:
return name == RESPONSE_FORMAT_TOOL_NAME
case _:
return False
def _openai_batch_jsonl_entry_to_vertex_anthropic_row(
custom_id: object,
messages: list[AllMessageValues],
request_params: Mapping[str, object],
model: str,
deployment_params: Mapping[str, object],
) -> Mapping[str, object]:
"""
Transforms an OpenAI chat batch entry into the Anthropic Messages row Vertex batches Claude with,
`{"custom_id": ..., "request": {"messages": [...], "anthropic_version": "vertex-2023-10-16", "max_tokens": ...}}`
"""
if custom_id is None:
raise ValueError("custom_id is required on Claude lines, since Claude output rows are matched back only by it")
config: Final = VertexAIAnthropicConfig()
bare_model: Final = model.rsplit("/", 1)[-1]
params: Final = {**deployment_params, **request_params}
drop_params: Final = drops_unsupported_params(params.get("drop_params"))
chat_params: Final = {
**get_non_default_completion_params(params),
**{key: value for key, value in params.items() if key in _CLAUDE_BATCH_OPENAI_PARAMS},
}
if any(map(_names_the_response_format_tool, iter_items(chat_params.get("tools") or ()))):
raise ValueError(
f"tool name {RESPONSE_FORMAT_TOOL_NAME!r} is reserved for response_format on Claude, "
"and batch output cannot tell a user tool by that name apart, so rename it"
)
optional_params: Final[Mapping[str, object]] = get_optional_params(
model=bare_model,
custom_llm_provider="vertex_ai",
drop_params=drop_params,
messages=messages,
**{ # pyright: ignore[reportArgumentType] # batch JSONL bodies are untyped; get_optional_params checks each param as chat does
key: value for key, value in chat_params.items() if key not in _CLAUDE_BATCH_ROW_FIXED_PARAMS
},
)
request_state: Final[dict[str, object]] = {"drop_params": drop_params}
anthropic_request: Final[Mapping[str, object]] = config.transform_request(
model=bare_model,
messages=messages,
optional_params={
**{key: value for key, value in optional_params.items() if key not in ("json_mode", "extra_body")},
"anthropic_version": VERTEX_ANTHROPIC_VERSION,
},
litellm_params=request_state,
headers={},
)
renamed_tools: Final = as_str_mapping(request_state.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY))
if renamed_tools:
raise ValueError(
f"tool names {sorted(map(str, renamed_tools.values()))} must match ^[a-zA-Z0-9_-]{{1,128}}$ for Claude on Vertex, "
"and batch output cannot map a renamed tool back, so rename these tools"
)
extra_body: Final = as_json_object(optional_params.get("extra_body"))
request: Final[Mapping[str, object]] = (
anthropic_request
if extra_body is None
else {
**anthropic_request,
**config.transform_extra_body(
extra_body=extra_body, request=anthropic_request, model=bare_model, litellm_params={}
),
}
)
return {"custom_id": str(custom_id), "request": request}
def _openai_batch_jsonl_entry_to_vertex_rows(
openai_entry: dict[str, Any],
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
batch_model: str | None = None,
deployment_params: Mapping[str, object] = MappingProxyType({}),
) -> tuple[Mapping[str, object], ...]:
"""
Transforms a single OpenAI JSONL batch entry into the Vertex rows it maps to.
`batch_model` is the deployment model the batch runs against and wins over the entry's own `body.model`,
which picks the row shape: Anthropic Messages for Claude, `GenerateContentRequest` otherwise.
jsonl body for vertex is {"request": <request_body>}
Example Vertex jsonl
{"request":{"contents": [{"role": "user", "parts": [{"text": "What is the relation between the following video and image samples?"}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/video/animals.mp4", "mimeType": "video/mp4"}}, {"fileData": {"fileUri": "gs://cloud-samples-data/generative-ai/image/cricket.jpeg", "mimeType": "image/jpeg"}}]}]}}
"""
entry_body: Final = openai_entry.get("body")
model: Final = batch_model or (str(entry_body.get("model") or "") if isinstance(entry_body, dict) else "")
is_anthropic_model: Final = is_vertex_anthropic_batch_model(model)
if _is_embeddings_batch_entry(openai_entry):
if is_anthropic_model:
raise VertexAIError(
status_code=400,
message=f"Vertex AI batch embeddings are not supported for Claude models ('{model}')",
)
return _openai_batch_jsonl_entry_to_vertex_embeddings_rows(openai_entry)
if not isinstance(openai_entry.get("body"), dict):
if not isinstance(entry_body, dict):
raise TypeError("`body` on batch requests must be a JSON object, but was missing or not an object")
openai_request_body: Final = openai_entry["body"]
chat_request_body: Final = (
@ -706,8 +861,21 @@ def _openai_batch_jsonl_entry_to_vertex_rows(
if _is_responses_batch_entry(openai_entry)
else openai_request_body
)
chat_messages: Final = map_developer_role_to_system_role(chat_request_body.get("messages", []))
if is_anthropic_model:
return (
_openai_batch_jsonl_entry_to_vertex_anthropic_row(
custom_id=openai_entry.get("custom_id"),
messages=chat_messages,
request_params={
key: value for key, value in chat_request_body.items() if key not in ("model", "messages")
},
model=model,
deployment_params=deployment_params,
),
)
vertex_request_body: Final = transform_request_body(
messages=map_developer_role_to_system_role(chat_request_body.get("messages", [])),
messages=chat_messages,
model=chat_request_body.get("model", ""),
optional_params=map_openai_to_vertex_params(chat_request_body),
custom_llm_provider="vertex_ai",
@ -859,15 +1027,24 @@ class _OpenAIToVertexBatchUploadStream(BaseFileUploadStream):
self,
openai_file_content: FileTypes,
map_openai_to_vertex_params: Callable[[dict[str, object]], dict[str, object]],
batch_model: str | None = None,
deployment_params: Mapping[str, object] = MappingProxyType({}),
) -> None:
self._openai_file_content = openai_file_content
self._map_openai_to_vertex_params = map_openai_to_vertex_params
self._batch_model = batch_model
self._deployment_params = deployment_params
def _iter_vertex_jsonl_chunks(self) -> Iterator[bytes]:
first = True
for lineno, entry in _iter_openai_jsonl_entries(self._openai_file_content):
try:
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(entry, self._map_openai_to_vertex_params)
vertex_rows = _openai_batch_jsonl_entry_to_vertex_rows(
entry,
self._map_openai_to_vertex_params,
batch_model=self._batch_model,
deployment_params=self._deployment_params,
)
except Exception as e:
raise VertexAIError(
status_code=400,
@ -923,16 +1100,31 @@ class _RawFileUploadStream(BaseFileUploadStream):
return _iter_raw_file_chunks(self._file_content)
def _deployment_batch_model(litellm_params: Mapping[str, object]) -> str | None:
model: Final = litellm_params.get("model")
return model.removeprefix("vertex_ai/") if isinstance(model, str) and model else None
def _publisher_model_path(raw_model: str) -> str:
if _PUBLISHER_MODEL_PATH_PATTERN.search(raw_model):
return raw_model
publisher: Final = "anthropic" if is_vertex_anthropic_batch_model(raw_model) else "google"
return f"publishers/{publisher}/models/{raw_model}"
def _managed_batch_object_name(raw_model: str, *, passthrough: bool) -> str:
endpoint_id: Final = get_vertex_ai_fine_tuned_endpoint_id(raw_model)
model_path: Final = (
f"endpoints/{endpoint_id}"
if endpoint_id is not None
else (raw_model if "publishers/google/models" in raw_model else f"publishers/google/models/{raw_model}")
)
safe_model_path: Final = sanitize_cloud_object_path(model_path, fallback="model")
model_path: Final = f"endpoints/{endpoint_id}" if endpoint_id is not None else _publisher_model_path(raw_model)
prefix: Final = _PASSTHROUGH_MANAGED_GCS_PREFIX if passthrough else VERTEX_AI_MANAGED_GCS_PREFIX
return f"{prefix}{safe_model_path}/{uuid.uuid4()}"
return f"{prefix}{_safe_managed_model_path(model_path)}/{uuid.uuid4()}"
def _safe_managed_model_path(model_path: str) -> str:
if not is_vertex_anthropic_batch_model(model_path):
return sanitize_cloud_object_path(model_path, fallback="model")
return "@".join(
sanitize_cloud_object_path(version_part, fallback="model") for version_part in model_path.split("@")
)
class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
@ -1036,10 +1228,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
fallback_filename="file",
)
def _get_configured_bucket_name(self, litellm_params: dict) -> str:
def _get_configured_bucket_name(self, litellm_params: Mapping[str, object]) -> str:
configured_bucket_name: Final = litellm_params.get("gcs_bucket_name") or litellm_params.get("bucket_name")
bucket_name: Final = (
litellm_params.get("gcs_bucket_name")
or litellm_params.get("bucket_name")
(configured_bucket_name if isinstance(configured_bucket_name, str) else None)
or os.getenv("GCS_BATCH_BUCKET_NAME")
or os.getenv("GCS_BUCKET_NAME")
)
@ -1053,7 +1245,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
api_key: str | None,
model: str,
optional_params: dict,
litellm_params: dict,
litellm_params: Mapping[str, object],
data: CreateFileRequest,
) -> str:
"""
@ -1077,11 +1269,10 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
raise ValueError("file is required")
if purpose is None:
raise ValueError("purpose is required")
configured_model: Final = litellm_params.get("model")
object_name = self.get_object_name(
file_data,
purpose,
deployment_model=configured_model if isinstance(configured_model, str) else None,
deployment_model=_deployment_batch_model(litellm_params),
passthrough=is_passthrough_batch_upload(data, litellm_params),
)
if object_prefix:
@ -1138,7 +1329,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
model: str,
create_file_data: CreateFileRequest,
optional_params: dict,
litellm_params: dict,
litellm_params: Mapping[str, object],
) -> bytes | str | dict:
"""
2 Cases:
@ -1170,6 +1361,8 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
body_stream=_OpenAIToVertexBatchUploadStream(
file_data,
partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params),
batch_model=_deployment_batch_model(litellm_params),
deployment_params=_claude_batch_deployment_params(litellm_params),
),
content_type="application/json",
)
@ -1220,7 +1413,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException:
return VertexAIError(status_code=status_code, message=error_message, headers=headers)
def _parse_gcs_uri(self, file_id: str, litellm_params: dict | None = None) -> tuple[str, str]:
def _parse_gcs_uri(self, file_id: str, litellm_params: Mapping[str, object] | None = None) -> tuple[str, str]:
"""
Validate a managed GCS file_id and return (bucket, url-encoded-object-path).
"""
@ -1238,7 +1431,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
litellm_params: Mapping[str, object],
) -> tuple[str, dict]:
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
@ -1268,7 +1461,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
self,
file_id: str,
optional_params: dict,
litellm_params: dict,
litellm_params: Mapping[str, object],
) -> tuple[str, dict]:
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
url: Final = f"https://storage.googleapis.com/storage/v1/b/{bucket}/o/{encoded_object}"
@ -1311,7 +1504,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
self,
file_content_request,
optional_params: dict,
litellm_params: dict,
litellm_params: Mapping[str, object],
) -> tuple[str, dict]:
file_id: Final = file_content_request.get("file_id", "")
bucket, encoded_object = self._parse_gcs_uri(file_id, litellm_params)
@ -1408,7 +1601,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
headers=MappingProxyType({**headers, "content-length": str(len(transformed_content))}),
)
if not _is_vertex_generate_content_batch_output_row(first_row):
if not _is_vertex_chat_batch_output_row(first_row):
return FileContentStreamingResult(stream_iterator=replayed_stream, headers=headers)
return FileContentStreamingResult(
@ -1492,10 +1685,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
# first line is not valid UTF-8/JSON) raises and falls through to the
# passthrough below, leaving the content untouched.
first_row: Final = _parse_vertex_batch_output_row(first_line)
if not (
_is_vertex_embeddings_batch_output_row(first_row)
or _is_vertex_generate_content_batch_output_row(first_row)
):
if not (_is_vertex_embeddings_batch_output_row(first_row) or _is_vertex_chat_batch_output_row(first_row)):
return content
context: Final = _new_vertex_batch_output_row_transform_context()
@ -1560,6 +1750,13 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig):
error_message=status,
)
if is_vertex_anthropic_batch_output_row(vertex_output):
return _vertex_anthropic_batch_output_row_to_openai(
custom_id=custom_id,
response=vertex_output.get("response"),
mock_httpx_response=mock_httpx_response,
)
# Transform successful response using existing transformation
vertex_response: Final = vertex_output.get("response", {})

View file

@ -7975,6 +7975,8 @@ async def atext_completion(*args, **kwargs) -> TextCompletionResponse | TextComp
model=model,
custom_llm_provider=custom_llm_provider,
stream_options=kwargs.get("stream_options"),
usage_injected_by_proxy="_litellm_strip_stream_usage" in kwargs
and kwargs["_litellm_strip_stream_usage"] is True,
)
else:
## OpenAI / Azure Text Completion Returns here
@ -8824,8 +8826,9 @@ def speech(
extra_headers: Final = kwargs.get("extra_headers", None)
model_info: Final = kwargs.get("model_info", None)
shared_session: Final = kwargs.get("shared_session", None)
caller_api_base: Final = api_base or base_url
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model, custom_llm_provider=custom_llm_provider, api_base=api_base or base_url
model=model, custom_llm_provider=custom_llm_provider, api_base=caller_api_base
)
kwargs.pop("tags", [])
@ -8880,13 +8883,13 @@ def speech(
custom_llm_provider=custom_llm_provider,
)
response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
if custom_llm_provider == "edenai":
litellm_params_dict["api_base"] = api_base
if custom_llm_provider in ("edenai", "dashscope") and text_to_speech_provider_config is not None:
litellm_params_dict["api_base"] = caller_api_base if custom_llm_provider == "dashscope" else api_base
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice if isinstance(voice, str) else None,
text_to_speech_provider_config=text_to_speech_provider_config or litellm.EdenAITextToSpeechConfig(),
text_to_speech_provider_config=text_to_speech_provider_config,
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,

View file

@ -121,6 +121,7 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType(
Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook,
Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception,
Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted,
Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry,
}
)

View file

@ -18346,25 +18346,31 @@
"dashscope/qwen-image-2.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.035,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-2.0-pro": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.075,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0": {
"litellm_provider": "dashscope",
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"output_cost_per_image": 0.03,
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen-image-3.0-pro": {
@ -18372,7 +18378,54 @@
"mode": "image_generation",
"source": "https://www.alibabacloud.com/help/en/model-studio/models",
"supported_endpoints": [
"/v1/images/generations"
"/v1/images/generations",
"/v1/images/edits"
]
},
"dashscope/qwen3-tts-flash": {
"input_cost_per_character": 1e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-flash": {
"input_cost_per_token": 2.3e-07,
"output_cost_per_token": 1.87e-06,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.1-tts-plus": {
"input_cost_per_character": 3e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-plus": {
"input_cost_per_character": 2e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/qwen-audio-3.0-tts-flash": {
"input_cost_per_character": 1.5e-05,
"litellm_provider": "dashscope",
"mode": "audio_speech",
"source": "https://www.alibabacloud.com/help/en/model-studio/model-pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"dashscope/wan3.0-video": {
@ -21382,7 +21435,7 @@
"cache_creation_input_token_cost": 3.74997e-06,
"cache_read_input_token_cost": 3.0002e-07,
"deprecation_date": "2026-10-09",
"input_cost_per_token": 2.9999900000000002e-06,
"input_cost_per_token": 2.99999e-06,
"input_dbu_cost_per_token": 4.2857e-05,
"litellm_provider": "databricks",
"max_input_tokens": 200000,
@ -21391,7 +21444,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 1.5000020000000002e-05,
"output_cost_per_token": 1.500002e-05,
"output_dbu_cost_per_token": 0.000214286,
"prompt_cache_min_tokens": 1024,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
@ -21504,6 +21557,7 @@
"databricks/databricks-deepseek-v4-flash-0731": {
"cache_creation_input_token_cost": 1.4e-07,
"cache_read_input_token_cost": 2.8e-08,
"deprecation_date": "2026-11-05",
"input_cost_per_token": 1.4e-07,
"input_dbu_cost_per_token": 2e-06,
"litellm_provider": "databricks",
@ -21516,7 +21570,7 @@
"mode": "chat",
"output_cost_per_token": 2.8e-07,
"output_dbu_cost_per_token": 4e-06,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text"
],
@ -21537,6 +21591,7 @@
"databricks/databricks-deepseek-v4-pro-0813": {
"cache_creation_input_token_cost": 1.31999e-06,
"cache_read_input_token_cost": 1.3202e-07,
"deprecation_date": "2026-10-30",
"input_cost_per_token": 1.31999e-06,
"input_dbu_cost_per_token": 1.8857e-05,
"litellm_provider": "databricks",
@ -21548,7 +21603,7 @@
"mode": "chat",
"output_cost_per_token": 3.95997e-06,
"output_dbu_cost_per_token": 5.6571e-05,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text"
],
@ -21735,7 +21790,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 1.5000020000000002e-05,
"output_cost_per_token": 1.500002e-05,
"output_cost_per_token_above_200k_tokens": 2.25e-05,
"output_dbu_cost_per_token": 0.000214286,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
@ -21994,7 +22049,7 @@
"databricks/databricks-gemma-3-12b": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_cost_per_token": 1.5001e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
"max_input_tokens": 128000,
@ -22117,7 +22172,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 9.999990000000002e-06,
"output_cost_per_token": 9.99999e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
@ -22140,7 +22195,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 9.999990000000002e-06,
"output_cost_per_token": 9.99999e-06,
"output_dbu_cost_per_token": 0.000142857,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
@ -22211,7 +22266,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 1.5000020000000002e-05,
"output_cost_per_token": 1.500002e-05,
"output_cost_per_token_above_272k_tokens": 2.25e-05,
"output_dbu_cost_per_token": 0.000214286,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
@ -22486,7 +22541,7 @@
"databricks/databricks-gpt-5-mini": {
"cache_creation_input_token_cost": 2.4997e-07,
"cache_read_input_token_cost": 2.499e-08,
"input_cost_per_token": 2.4997000000000006e-07,
"input_cost_per_token": 2.4997e-07,
"input_dbu_cost_per_token": 3.571e-06,
"litellm_provider": "databricks",
"max_input_tokens": 272000,
@ -22495,7 +22550,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 1.9999700000000004e-06,
"output_cost_per_token": 1.99997e-06,
"output_dbu_cost_per_token": 2.8571e-05,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
@ -22518,7 +22573,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 3.9998000000000007e-07,
"output_cost_per_token": 3.9998e-07,
"output_dbu_cost_per_token": 5.714000000000001e-06,
"source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
"supports_function_calling": true,
@ -22528,7 +22583,7 @@
"databricks/databricks-gpt-oss-120b": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_cost_per_token": 1.5001e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
"max_input_tokens": 131072,
@ -22553,7 +22608,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 3.0001999999999996e-07,
"output_cost_per_token": 3.0002e-07,
"output_dbu_cost_per_token": 4.285999999999999e-06,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
@ -22586,7 +22641,7 @@
"databricks/databricks-gte-large-en": {
"cache_creation_input_token_cost": 1.2999e-07,
"cache_read_input_token_cost": 1.2999e-07,
"input_cost_per_token": 1.2999000000000001e-07,
"input_cost_per_token": 1.2999e-07,
"input_dbu_cost_per_token": 1.857e-06,
"litellm_provider": "databricks",
"max_input_tokens": 8192,
@ -22602,6 +22657,7 @@
"databricks/databricks-inkling": {
"cache_creation_input_token_cost": 1.00002e-06,
"cache_read_input_token_cost": 1.7003e-07,
"deprecation_date": "2026-10-30",
"input_cost_per_token": 1.00002e-06,
"input_dbu_cost_per_token": 1.4286e-05,
"litellm_provider": "databricks",
@ -22612,7 +22668,7 @@
"mode": "chat",
"output_cost_per_token": 4.04999e-06,
"output_dbu_cost_per_token": 5.7857e-05,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"source": "https://docs.databricks.com/aws/en/machine-learning/foundation-model-apis/supported-models",
"supported_modalities": [
"text",
"image"
@ -22677,7 +22733,7 @@
"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."
},
"mode": "chat",
"output_cost_per_token": 1.5000300000000002e-06,
"output_cost_per_token": 1.50003e-06,
"output_dbu_cost_per_token": 2.1429e-05,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"supports_tool_choice": true
@ -22685,7 +22741,7 @@
"databricks/databricks-meta-llama-3-1-8b-instruct": {
"cache_creation_input_token_cost": 1.5001e-07,
"cache_read_input_token_cost": 1.5001e-07,
"input_cost_per_token": 1.5000999999999998e-07,
"input_cost_per_token": 1.5001e-07,
"input_dbu_cost_per_token": 2.1429999999999996e-06,
"litellm_provider": "databricks",
"max_input_tokens": 200000,
@ -22694,7 +22750,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 4.5003000000000007e-07,
"output_cost_per_token": 4.5003e-07,
"output_dbu_cost_per_token": 6.429000000000001e-06,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving"
},
@ -22710,7 +22766,7 @@
"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."
},
"mode": "chat",
"output_cost_per_token": 1.5000300000000002e-06,
"output_cost_per_token": 1.50003e-06,
"output_dbu_cost_per_token": 2.1429e-05,
"source": "https://www.databricks.com/product/pricing/foundation-model-serving",
"supports_tool_choice": true
@ -27741,6 +27797,7 @@
},
"fireworks_ai/accounts/fireworks/models/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"deprecation_date": "2026-08-14",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
@ -28189,6 +28246,7 @@
},
"fireworks_ai/deepseek-v4-flash": {
"cache_read_input_token_cost": 2.8e-08,
"deprecation_date": "2026-08-14",
"input_cost_per_token": 1.4e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
@ -48560,13 +48618,13 @@
"supports_reasoning": true
},
"together_ai/Qwen/Qwen3.7-Max": {
"cache_read_input_token_cost": 5e-07,
"input_cost_per_token": 2.5e-06,
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 1.5e-06,
"litellm_provider": "together_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 7.5e-06,
"output_cost_per_token": 4.5e-06,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
@ -65073,7 +65131,7 @@
"wandb/zai-org/GLM-5.2": {
"supports_reasoning": true,
"max_output_tokens": 262144,
"max_input_tokens": 1049000,
"max_input_tokens": 262000,
"input_cost_per_token": 7.6e-07,
"output_cost_per_token": 2.42e-06,
"cache_read_input_token_cost": 1.4e-07,
@ -65081,7 +65139,7 @@
"litellm_provider": "wandb",
"mode": "chat",
"supports_vision": false,
"source": "https://wandb.ai/site/pricing/tokens/"
"source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
},
"deepinfra/openai/gpt-oss-120b-Turbo": {
"max_output_tokens": 131072,
@ -72330,6 +72388,28 @@
"source": "https://docs.typesafe.ai/models",
"supports_box_questions": true
},
"eu_jev/eu-jeff-1.0.0": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"eu_jev/eu-jeff-latest": {
"input_cost_per_token": 4.15e-08,
"litellm_provider": "eu_jev",
"max_input_tokens": 32768,
"mode": "evaluation",
"output_cost_per_token": 0.0,
"source": "https://eu-jev.com/docs",
"supported_endpoints": [
"/v1/systemone"
]
},
"wandb/deepseek-ai/DeepSeek-V4.1-Flash": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 2e-07,
@ -78558,6 +78638,272 @@
"supports_xhigh_reasoning_effort": true,
"use_openai_responses_path": true
},
"clf_ai_gateway/glm-5.3": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 5.6e-07,
"output_cost_per_token": 1.76e-06,
"cache_read_input_token_cost": 1.04e-07,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/glm-5.3-flash": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 6e-08,
"output_cost_per_token": 2e-07,
"cache_read_input_token_cost": 1.2e-08,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high",
"xhigh",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"supports_vision": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/glm-5.2": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 262144,
"max_output_tokens": 131072,
"input_cost_per_token": 5.6e-07,
"output_cost_per_token": 1.76e-06,
"cache_read_input_token_cost": 1.04e-07,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high",
"xhigh",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/glm-4.7-flash": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"input_cost_per_token": 2.4e-08,
"output_cost_per_token": 1.6e-07,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high"
],
"supports_response_schema": true,
"supports_system_messages": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/deepseek-v4-pro": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 5.28e-07,
"output_cost_per_token": 1.584e-06,
"cache_read_input_token_cost": 1.7e-08,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high",
"xhigh",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/deepseek-v4-flash": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.76e-07,
"output_cost_per_token": 5.28e-07,
"cache_read_input_token_cost": 5e-09,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high",
"xhigh",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/deepseek-v4.1-flash": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"input_cost_per_token": 1.2e-07,
"output_cost_per_token": 4.8e-07,
"cache_read_input_token_cost": 2e-09,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"none",
"low",
"medium",
"high",
"xhigh",
"max"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"supports_vision": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/kimi-k2.6": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 262144,
"max_output_tokens": 131072,
"input_cost_per_token": 3.8e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 6.4e-08,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"none",
"low",
"medium",
"high"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"supports_vision": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/kimi-k2.7-code": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 262144,
"max_output_tokens": 131072,
"input_cost_per_token": 3.8e-07,
"output_cost_per_token": 1.6e-06,
"cache_read_input_token_cost": 7.6e-08,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"high"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"supports_vision": true,
"source": "https://clfaigateway.dev/models"
},
"clf_ai_gateway/qwen3.8-27b": {
"metadata": {
"comment": "Billed price checked 2026-10-29, which the gateway shows as 60% off a list price 2.5x higher. Update the rows when the discount ends."
},
"max_input_tokens": 262144,
"max_output_tokens": 131072,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 1.28e-06,
"cache_read_input_token_cost": 2e-08,
"litellm_provider": "clf_ai_gateway",
"mode": "chat",
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"reasoning_effort_levels": [
"low",
"medium",
"xhigh"
],
"supports_response_schema": true,
"supports_system_messages": true,
"supports_prompt_caching": true,
"supports_vision": true,
"source": "https://clfaigateway.dev/models"
},
"kyma/claude-fable-5": {
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
@ -80538,5 +80884,22 @@
"/v1/messages"
],
"supports_native_streaming": true
},
"fireworks_ai/accounts/fireworks/models/ultramarine-1-flash-us": {
"cache_read_input_token_cost": 4.5e-08,
"cache_read_input_token_cost_priority": 5.625e-08,
"input_cost_per_token": 2.25e-07,
"input_cost_per_token_priority": 2.8125e-07,
"litellm_provider": "fireworks_ai",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 7.5e-07,
"output_cost_per_token_priority": 9.375e-07,
"source": "https://docs.fireworks.ai/serverless/pricing",
"supports_function_calling": false,
"supports_response_schema": true,
"supports_tool_choice": false,
"supports_vision": true
}
}

View file

@ -827,6 +827,32 @@
"generateContent": false
}
},
"clf_ai_gateway": {
"display_name": "CLF AI Gateway (`clf_ai_gateway`)",
"url": "https://docs.litellm.ai/docs/providers/clf_ai_gateway",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"cloudflare": {
"display_name": "Cloudflare AI Workers (`cloudflare`)",
"url": "https://docs.litellm.ai/docs/providers/cloudflare_workers",
@ -1148,9 +1174,9 @@
"text_completion": true,
"embeddings": false,
"image_generations": true,
"image_edits": false,
"image_edits": true,
"audio_transcriptions": false,
"audio_speech": false,
"audio_speech": true,
"rerank": false,
"ocr": false,
"search": false,
@ -2241,6 +2267,32 @@
"generateContent": false
}
},
"eu_jev": {
"display_name": "eu/jev (`eu_jev`)",
"url": "https://docs.litellm.ai/docs/pass_through/eu_jev",
"endpoints": {
"chat_completions": false,
"responses": false,
"text_completion": false,
"embeddings": false,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": false,
"interactions": false,
"systemone": true,
"a2a": false,
"generateContent": false
}
},
"laya": {
"display_name": "Laya (`laya`)",
"url": "https://docs.litellm.ai/docs/auto_router/decision_classifiers",

View file

@ -238,6 +238,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
"/tinyfish/",
"/transcribe",
"/typesafe/",
"/eu_jev/",
"/laya/",
"/bespoke/",
"/openrouter/",

View file

@ -18332,6 +18332,7 @@
{
"enum": [
"block",
"mask",
"monitor",
"inject_system_message"
],
@ -18342,7 +18343,7 @@
}
],
"default": "block",
"description": "Action to take when content is flagged: 'block' (raise exception), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"description": "Action to take when content is flagged: 'block' (raise exception), 'mask' (mask PII-only detections and allow, block anything else), 'monitor' (log only), or 'inject_system_message' (append an advisory system message and let the LLM decide)",
"title": "On Flagged"
},
"on_flagged_action": {
@ -23169,6 +23170,39 @@
"title": "CachedTokensDetails",
"type": "object"
},
"ChoiceAnswer": {
"additionalProperties": true,
"properties": {
"choice": {
"title": "Choice",
"type": "string"
},
"confidence": {
"title": "Confidence",
"type": "number"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"choice",
"confidence",
"probabilities"
],
"title": "ChoiceAnswer",
"type": "object"
},
"Click": {
"additionalProperties": true,
"description": "A click action.",
@ -23933,6 +23967,112 @@
"title": "ErrorResponse",
"type": "object"
},
"EuJevDecisionResponse": {
"properties": {
"answers": {
"additionalProperties": {
"discriminator": {
"mapping": {
"choice": "#/components/schemas/ChoiceAnswer",
"noul": "#/components/schemas/NoulAnswer",
"score": "#/components/schemas/ScoreAnswer"
},
"propertyName": "type"
},
"oneOf": [
{
"$ref": "#/components/schemas/ChoiceAnswer"
},
{
"$ref": "#/components/schemas/NoulAnswer"
},
{
"$ref": "#/components/schemas/ScoreAnswer"
}
]
},
"description": "One answer per question, under the same names",
"title": "Answers",
"type": "object"
},
"meta": {
"$ref": "#/components/schemas/EuJevMeta"
},
"model": {
"description": "The release that answered, such as eu-jeff-1.0.0",
"title": "Model",
"type": "string"
},
"usage": {
"$ref": "#/components/schemas/EuJevUsage"
}
},
"required": [
"model",
"answers",
"usage",
"meta"
],
"title": "EuJevDecisionResponse",
"type": "object"
},
"EuJevMeta": {
"properties": {
"cost_eur": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "The exact charge in euros, on authenticated requests",
"title": "Cost Eur"
},
"latency_ms": {
"description": "Server-side time for token quoting, credit reservation and inference",
"title": "Latency Ms",
"type": "integer"
},
"mode": {
"const": "live",
"title": "Mode",
"type": "string"
},
"request_id": {
"title": "Request Id",
"type": "string"
}
},
"required": [
"request_id",
"mode",
"latency_ms"
],
"title": "EuJevMeta",
"type": "object"
},
"EuJevUsage": {
"properties": {
"input_tokens": {
"minimum": 0.0,
"title": "Input Tokens",
"type": "integer"
},
"output_tokens": {
"minimum": 0.0,
"title": "Output Tokens",
"type": "integer"
}
},
"required": [
"input_tokens",
"output_tokens"
],
"title": "EuJevUsage",
"type": "object"
},
"FileSearchTool": {
"additionalProperties": true,
"description": "A tool that searches for relevant content from uploaded files.\n\nLearn more about the [file search tool](https://platform.openai.com/docs/guides/tools-file-search).",
@ -25593,6 +25733,26 @@
"title": "NamespaceTool",
"type": "object"
},
"NoulAnswer": {
"additionalProperties": true,
"properties": {
"noul": {
"title": "Noul",
"type": "number"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"noul"
],
"title": "NoulAnswer",
"type": "object"
},
"OperationCreateFile": {
"additionalProperties": true,
"description": "Instruction describing how to create a file via the apply_patch tool.",
@ -28865,6 +29025,59 @@
"title": "Result",
"type": "object"
},
"ScoreAnswer": {
"additionalProperties": true,
"properties": {
"confidence": {
"title": "Confidence",
"type": "number"
},
"legend": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
]
},
"title": "Legend",
"type": "object"
},
"probabilities": {
"additionalProperties": {
"type": "number"
},
"title": "Probabilities",
"type": "object"
},
"score": {
"title": "Score",
"type": "number"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"score",
"confidence",
"legend",
"probabilities"
],
"title": "ScoreAnswer",
"type": "object"
},
"Screenshot": {
"additionalProperties": true,
"description": "A screenshot action.",
@ -31757,6 +31970,315 @@
]
}
},
"/eu_jev/v1/systemone": {
"post": {
"description": "Forward a request to eu/jev, an EU-hosted decision API, with `EU_JEV_API_KEY`.\n\n`POST /eu_jev/v1/systemone` sends a `state` and one to eight named `questions` (`choice`, `noul` or `score`)\nand returns one answer per question name, with spend booked from the reported `usage`. `custom_body` and\n`stream` are rejected with a 400. A pass-through endpoint configured at this path keeps serving it.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/eu_jev)",
"operationId": "eu_jev_proxy_route_eu_jev_v1_systemone_post",
"requestBody": {
"content": {
"application/json": {
"example": {
"model": "eu-jeff-latest",
"questions": {
"frustration": {
"criteria": [
"calm",
"mildly annoyed",
"angry"
],
"instructions": "How frustrated is the customer?",
"type": "score"
},
"is_refund_request": {
"instructions": "Is the customer asking for a refund?",
"type": "noul"
},
"team": {
"criteria": {
"billing": "Payments, invoices, and refunds",
"support": "Technical issues and bugs"
},
"instructions": "Which team should handle this?",
"type": "choice"
}
},
"state": "I was charged twice. Can I get a refund?"
},
"schema": {
"properties": {
"model": {
"default": "eu-jeff-latest",
"description": "eu-jeff-latest follows the current release, eu-jeff-1.0.0 pins it",
"title": "Model",
"type": "string"
},
"questions": {
"additionalProperties": {
"discriminator": {
"propertyName": "type"
},
"oneOf": [
{
"properties": {
"criteria": {
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"description": "Two to 64 options, option name to description. A null description uses the option name",
"maxProperties": 64,
"minProperties": 2,
"title": "Criteria",
"type": "object"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "choice",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevChoiceQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"anyOf": [
{
"additionalProperties": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
]
},
"propertyNames": {
"enum": [
"true",
"false"
]
},
"type": "object"
},
{
"type": "null"
}
],
"description": "Optional descriptions of what true and false mean",
"title": "Criteria"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "noul",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions"
],
"title": "EuJevNoulQuestion",
"type": "object"
},
{
"properties": {
"criteria": {
"description": "Two to 10 level descriptions, ordered from low to high",
"items": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference"
},
"maxItems": 10,
"minItems": 2,
"title": "Criteria",
"type": "array"
},
"instructions": {
"anyOf": [
{
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
},
{
"type": "null"
}
],
"description": "Free text, or a JSON object, array or null that eu/jev serializes to text before inference",
"title": "Instructions"
},
"type": {
"const": "score",
"title": "Type",
"type": "string"
}
},
"required": [
"type",
"instructions",
"criteria"
],
"title": "EuJevScoreQuestion",
"type": "object"
}
]
},
"description": "One to eight questions, keyed by names of up to 80 characters",
"maxProperties": 8,
"minProperties": 1,
"propertyNames": {
"maxLength": 80,
"minLength": 1
},
"title": "Questions",
"type": "object"
},
"state": {
"anyOf": [
{
"maxLength": 100000,
"minLength": 1,
"type": "string"
},
{
"additionalProperties": true,
"type": "object"
},
{
"items": {},
"type": "array"
}
],
"description": "The content to evaluate, a string of up to 100,000 characters or a JSON object or array",
"title": "State"
}
},
"required": [
"state",
"questions"
],
"title": "EuJevDecisionRequest",
"type": "object"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/EuJevDecisionResponse"
}
}
},
"description": "Successful Response"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Eu Jev Proxy Route",
"tags": [
"llm_passthrough"
]
}
},
"/fal_ai/{endpoint}": {
"delete": {
"operationId": "fal_ai_proxy_route_fal_ai__endpoint__delete",

View file

@ -3177,11 +3177,18 @@ from litellm.types.proxy.auth.user_api_key_auth import (
)
class AdminMCPPermissions(LiteLLMPydanticObjectBase):
model_config = ConfigDict(frozen=True)
tools: tuple[str, ...]
action_allowed: bool | None = None
class UserInfoResponse(LiteLLMPydanticObjectBase):
user_id: str | None
user_info: dict | BaseModel | None
keys: list
teams: list
admin_mcp_permissions: AdminMCPPermissions | None = None
class UserInfoV2Response(LiteLLMPydanticObjectBase):

View file

@ -1,20 +1,21 @@
import os
import re
from collections.abc import AsyncGenerator, Mapping
from collections.abc import AsyncGenerator, Iterator, Mapping
from contextlib import asynccontextmanager
from contextvars import ContextVar
from typing import Final
from urllib.parse import urlsplit
from urllib.parse import unquote, urlsplit
from fastapi import FastAPI
from fastapi import FastAPI, HTTPException, Request
from fastapi.routing import APIRoute
from pydantic import TypeAdapter
from starlette.datastructures import Headers
from starlette.requests import Request
from starlette.routing import Mount
from starlette.types import ASGIApp, Receive, Scope, Send
from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
from litellm.proxy._types import SpecialHeaders
from litellm.proxy._types import AdminMCPPermissions, SpecialHeaders, UserAPIKeyAuth
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.middleware.admission_control_middleware import ADMISSION_LEASE_SCOPE_KEY
_REQUEST_HEADERS: Final = frozenset(
@ -58,6 +59,91 @@ _RESERVED_KEY_HEADERS: Final = (
_SETTINGS: Final = TypeAdapter(Mapping[str, object])
_IDENTITY_MAPPINGS: Final = TypeAdapter(tuple[dict[str, object], ...] | dict[str, object] | None)
_OAUTH_MAPPINGS: Final = TypeAdapter(dict[str, str])
_ROUTE_GRANTS: Final = TypeAdapter(tuple[str, ...])
def _admin_route_allowed(auth: UserAPIKeyAuth, request: Request, method: str, path: str) -> bool:
target: Final = Request({**request.scope, "method": method, "path": path, "raw_path": path.encode()})
try:
RouteChecks.should_call_route(route=path, valid_token=auth, request=target)
except HTTPException:
return False
return True
def _admin_routes(request: Request) -> Iterator[tuple[tuple[str, str], APIRoute]]:
app: Final = request.app
if not isinstance(app, FastAPI):
return
for route in app.routes:
if not isinstance(route, APIRoute):
continue
for method in route.methods:
yield (method, route.path_format), route
def _admin_grant_paths(route: APIRoute, grant: str) -> Iterator[str]:
if route.path_regex.fullmatch(grant) is not None:
yield grant
prefix: Final = grant[:-1] if grant.endswith("*") else grant + "/"
representative: Final = re.sub(r"\{[^}]+\}", "_", route.path_format)
# A matching suffix supplies the rest of a concrete route for prefix grants,
# including grants ending inside a static segment or a path parameter.
for candidate in (prefix + representative[offset:] for offset in range(len(representative) + 1)):
if route.path_regex.fullmatch(candidate) is not None:
yield candidate
def _admin_grant_allowed(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute, grant: str) -> bool:
return any(_admin_route_allowed(auth, request, method, path) for path in _admin_grant_paths(route, grant))
def _admin_tool_visible(auth: UserAPIKeyAuth, request: Request, method: str, route: APIRoute) -> bool:
try:
RouteChecks.check_route_enabled(route.path)
except HTTPException:
return False
if _admin_route_allowed(auth, request, method, route.path):
return True
grants: Final = _ROUTE_GRANTS.validate_python(getattr(auth, "allowed_routes", None) or ())
return bool(route.param_convertors) and any(
_admin_grant_allowed(auth, request, method, route, grant) for grant in grants if grant.startswith("/")
)
def get_admin_mcp_permissions(
auth: UserAPIKeyAuth, request: Request, tool: str | None = None, path: str | None = None
) -> AdminMCPPermissions:
try:
from litellm_admin_mcp.catalog import ( # pyright: ignore[reportMissingTypeStubs] # connector has no py.typed marker
BY_NAME,
OPERATIONS,
)
except ImportError as exc:
raise HTTPException(
status_code=503, detail="Admin MCP permissions require the admin-mcp dependency group (Python 3.12+)"
) from exc
routes: Final = dict(_admin_routes(request))
allowed: Final = tuple(
operation.name
for operation in OPERATIONS
if (route := routes.get((operation.method, operation.path))) is not None
and _admin_tool_visible(auth, request, operation.method, route)
)
if tool is None:
return AdminMCPPermissions(tools=allowed)
operation: Final = BY_NAME.get(tool)
if operation is None or tool not in allowed:
return AdminMCPPermissions(tools=allowed, action_allowed=False)
if path is None:
return AdminMCPPermissions(tools=allowed, action_allowed=True)
registered: Final = routes[(operation.method, operation.path)]
decoded_path: Final = unquote(path)
action_allowed: Final = registered.path_regex.fullmatch(decoded_path) is not None and _admin_route_allowed(
auth, request, operation.method, decoded_path
)
return AdminMCPPermissions(tools=allowed, action_allowed=action_allowed)
def _configured_key_header() -> bytes | None:

View file

@ -30,6 +30,7 @@ from litellm.proxy.auth.auth_utils import ( # noqa: F401 # legacy module expor
)
from litellm.proxy.auth.model_access_denied import ModelAccessDeniedHTTPException
from litellm.proxy.auth.resolvers.exceptions import KeyNotFoundError
from litellm.proxy.common_utils.openai_error_payload import openai_error_param
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.types.services import ServiceTypes
@ -61,14 +62,14 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
message=str(e.detail),
internal_message=e.internal_message,
type=ProxyErrorTypes.auth_error,
param="None",
param=None,
code=e.status_code,
)
if isinstance(e, HTTPException):
return ProxyException(
message=getattr(e, "detail", f"Authentication Error({e})"),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
)
if isinstance(e, ProxyException):
@ -78,7 +79,7 @@ def _as_proxy_exception(e: Exception) -> ProxyException:
return ProxyException(
message="Authentication Error, " + str(e),
type=ProxyErrorTypes.auth_error,
param=getattr(e, "param", "None"),
param=openai_error_param(e),
code=status.HTTP_401_UNAUTHORIZED,
)

View file

@ -33,6 +33,7 @@ from litellm.litellm_core_utils.url_utils import (
from litellm.llms.azure.passthrough.transformation import azure_router_model_in_endpoint
from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
from litellm.proxy._types import *
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.http_parsing_utils import extract_nested_form_metadata
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_ENDPOINT_MARKER,
@ -884,7 +885,6 @@ def route_in_additonal_public_routes(current_route: str):
public_routes: ["LiteLLMRoutes.public_routes", "/spend/calculate", "/api/*"]
```
"""
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.proxy_server import general_settings, premium_user
try:
@ -2175,6 +2175,13 @@ def _extract_model_candidates_from_request(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return _dedupe_model_candidates((f"{provider}/{decision_model}",))
if RouteChecks.is_builtin_eu_jev_decision_route(route=route):
from litellm.llms.eu_jev.common_utils import eu_jev_registry_model
requested_model: Final = TypeAdapter(Mapping[str, object]).validate_python(request_data).get("model")
if requested_model is not None and not isinstance(requested_model, str):
raise HTTPException(status_code=400, detail="eu_jev model must be a string")
return [f"eu_jev/{eu_jev_registry_model(requested_model)}"]
if route == "/cost/predict-cache":
prediction_models: Final = _cache_prediction_model_candidates(request_data, llm_router, team_id) # pyright: ignore[reportUnknownArgumentType] # the typed reader validates each deployment ID from this legacy payload
return _dedupe_model_candidates(prediction_models)

View file

@ -225,7 +225,7 @@ def jwks_unavailable_exception(error: JWKSUnreachableError) -> ProxyException:
f"unreachable, so the JWT signature could not be verified. Please retry shortly. Error: {error}"
),
type=ProxyErrorTypes.auth_provider_unavailable,
param="None",
param=None,
code=status.HTTP_503_SERVICE_UNAVAILABLE,
)

Some files were not shown because too many files have changed in this diff Show more