scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

Merge branch 'litellm_internal_staging' into chore/update-singulr-guardrails-api-contract

Browse source
This commit is contained in:
aniket-kardile 2026-08-24 15:51:24 +05:30 • committed by GitHub
commit 4315d8818f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2151 changed files with 33816 additions and 21071 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -651,126 +651,6 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -858,7 +738,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -985,7 +865,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -1030,7 +910,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1074,7 +954,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -1120,7 +1000,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -1211,7 +1091,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1255,7 +1135,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1274,40 +1154,6 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_enterprise_tests:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
batches_testing:
docker:
- *python312_image
@ -1333,7 +1179,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1377,7 +1223,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1422,7 +1268,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1501,7 +1347,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1546,7 +1392,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1599,7 +1445,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -3105,12 +2951,6 @@ workflows:
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:
@ -3167,8 +3007,6 @@ workflows:
filters: *main_branches
- search_testing:
filters: *main_branches
- litellm_mapped_enterprise_tests:
filters: *main_branches
- batches_testing:
filters: *main_branches
- litellm_utils_testing:
@ -3191,7 +3029,6 @@ workflows:
- guardrails_testing
- ocr_testing
- search_testing
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

6
.github/CODEOWNERS vendored
View file

@ -1,5 +1,7 @@
/ui/ @yuneng-jiang @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
/ui/ @yuneng-berri @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
/ui/litellm-dashboard/src/lib/http/schema.d.ts
/model_prices_and_context_window.json @mateo-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri
/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
/.github/CODEOWNERS @yuneng-berri

31
.github/actions/cache-cargo-build/action.yml vendored Normal file
View file

@ -0,0 +1,31 @@
name: "Cache the Rust build"
description: >-
Cache the Cargo registry and target directory the root package's build needs,
so only the first job on a given Cargo.lock compiles the bridge from scratch.
litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
in release mode before it can produce a wheel. `uv sync` therefore pays a full
build in every job that installs the workspace: measured at 2m40s per unit shard
on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
it, because the uv cache holds wheels uv downloads rather than wheels it builds,
and a path dependency whose source moves every commit could never hit that cache
anyway. Cargo rebuilds only what changed when its target directory survives, so a
warm job pays for the bridge crate alone.
The key namespace is separate from test-rust.yml's. Both cache the same directory,
but that workflow fills it with debug and clippy artifacts, which a release build
cannot reuse, and a shared key would let whichever ran first deny the other a save.
runs:
using: composite
steps:
- name: Restore the Cargo registry and target directory
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-release-

46
.github/ci-coverage-allowlist.yml vendored
View file

@ -5,24 +5,21 @@ description: >-
test_paths:
- reason: >-
The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
2026-08-20 by collecting the directory under each job's own selector: 118 tests across
these eight files are selected by none of them. Listed so the gap is a decision rather
than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
when tests/local_testing is ported off CircleCI, where the keyless part of this suite
belongs in a real job
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
split that porting tests/local_testing off CircleCI will force, not a job that is red by
construction
paths:
- tests/local_testing/test_cache_preset_key.py
- tests/local_testing/test_caching.py
- tests/local_testing/test_caching_handler.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- tests/local_testing/test_prompt_caching.py
- tests/local_testing/test_responses_stream_cache_keys.py
- tests/local_testing/test_unit_test_caching.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds
@ -48,16 +45,6 @@ test_paths:
choice it informed is settled
paths:
- tests/code_coverage_tests/test_aio_http_image_conversion.py
- reason: >-
The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
their bodies into the live file of the same name. This one cannot follow either route yet:
its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
counterpart while 25 assertions fail against today's code, so what survives that rewrite
is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
behaviours still hold
paths:
- tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
- reason: >-
No job invokes this suite and its files mix pure transformation tests with ones driving live
vendor vector stores, so assigning them needs a per-file decision
@ -102,17 +89,6 @@ test_paths:
- tests/integration/sandbox/test_e2b_sandbox.py
- tests/integration/test_oci_integration.py
- tests/integration/test_oci_proxy_integration.py
- reason: >-
A unit test for the proxy-extras package that no job invokes, while the package's other
tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
this file requires. It also matches those keywords inside SQL comments, so two further
migrations are reported that are in fact fine. Wiring it up means deciding what to do about
the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
changed one breaks migrate deploy for existing installs
paths:
- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
dockerfiles:
- reason: >-

16
.github/scripts/assert_ci_coverage.py vendored
View file

@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
)
def _workflow_named_tokens() -> frozenset[str]:
"""Test tokens a GitHub Actions job names directly.
A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
workflow names it, so the slice check has to credit those the same way the census does.
"""
return _invoked_test_tokens(
scalar
for path in _config_files()
if path != CIRCLECI_CONFIG
for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
)
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
slices: Final = _slices()
named_by_workflow: Final = _workflow_named_tokens()
globbed: Final = tuple(
path
for path in _test_files()
@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
)
for path in globbed
if not allowlist.covers_test(path)
and not any(_token_covers(token, path) for token in named_by_workflow)
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
)

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,7 +27,7 @@ on:
default: 20
job-timeout-minutes:
description: >-
Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
the per-step ceilings on the setup steps below, and 5 for the runner
overhead the job clock charges but no step owns (job init, step
transitions, post-job cleanup). That headroom is what makes the test
@ -36,7 +36,7 @@ on:
arithmetic, so the sum is passed in rather than computed.
required: false
type: number
default: 55
default: 60
max-failures:
description: "Stop after this many failures"
required: false
@ -103,6 +103,11 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
@ -144,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -156,7 +161,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

4
.github/workflows/check-ui-api-types.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

2
.github/workflows/image-scan.yml vendored
View file

@ -17,6 +17,8 @@ on:
- backend/Dockerfile
- backend/main.py
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock

3
.github/workflows/mutation-test.yml vendored
View file

@ -53,6 +53,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml

3
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -43,6 +43,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries

3
.github/workflows/test-code-quality.yml vendored
View file

@ -56,6 +56,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras

15
.github/workflows/test-linting.yml vendored
View file

@ -67,6 +67,17 @@ jobs:
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |
@ -78,6 +89,10 @@ jobs:
run: |
uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

6
.github/workflows/test-mcp.yml vendored
View file

@ -47,6 +47,10 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
@ -56,4 +60,4 @@ jobs:
- name: Run MCP tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

145
.github/workflows/test-postgres.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi

3
.github/workflows/test-terraform-provider.yml vendored
View file

@ -88,6 +88,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

4
.github/workflows/test-unit-documentation.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

51
.github/workflows/test-unit.yml vendored
View file

@ -55,7 +55,7 @@ jobs:
workers: 2
reruns: 1
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: enterprise-routing
artifact-name: enterprise-routing
@ -67,7 +67,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: integrations
artifact-name: integrations
@ -75,7 +75,7 @@ jobs:
workers: 2
reruns: 3
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: Vertex AI
artifact-name: llm-vertex-ai
@ -83,7 +83,7 @@ jobs:
workers: 1
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: All Other Providers
artifact-name: llm-other-providers
@ -91,7 +91,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: misc
artifact-name: misc
@ -122,7 +122,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-auth
artifact-name: proxy-auth
@ -134,7 +134,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-endpoints
artifact-name: proxy-endpoints
@ -168,10 +168,10 @@ jobs:
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-server
artifact-name: proxy-server
@ -179,7 +179,7 @@ jobs:
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
job-timeout-minutes: 100
- shard: proxy-infra
artifact-name: proxy-infra
@ -195,11 +195,40 @@ jobs:
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: proxy-extras
artifact-name: proxy-extras
test-path: "tests/litellm-proxy-extras"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: enterprise-package
artifact-name: enterprise-package
test-path: "tests/enterprise"
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
- shard: responses-caching-types
artifact-name: responses-caching-types
test-path: >-
@ -209,7 +238,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}

3
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -47,6 +47,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

2
basedpyright-code-budget.json
View file

@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1823
"limit": 1822
},
"reportRedeclaration": {
"limit": 8

1
ci_cd/generate_model_prices_schema.py
View file

@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

1
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -21,6 +21,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BannedKeywords(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self):
banned_keywords_list = litellm.banned_keywords_list

1
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -18,6 +18,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BlockedUserList(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self, prisma_client: Optional[PrismaClient]):
self.prisma_client = prisma_client

10
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -966,6 +966,16 @@ class CheckBatchCost:
)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._finalize_unbilled_terminal_job(job, response)
# Record polling run metrics (always, even if nothing was processed)

87
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -45,6 +45,8 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
FILE_LIST_CONTINUATION_CHUNK_SIZE,
MAX_FILE_LIST_LIMIT,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
ensure_batch_response_managed_file_ids,
@ -54,6 +56,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
map_raw_file_ids_to_unified,
normalize_mime_type_for_provider,
resolve_managed_output_file_model_name,
validate_file_list_limit,
validate_file_list_purpose,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
@ -63,9 +67,9 @@ from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccess
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileListPage,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
ResponsesAPIResponse,
)
from litellm.types.utils import (
@ -144,7 +148,14 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
async def find_many(
self,
where: Mapping[str, object],
take: int = ...,
order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
cursor: Mapping[str, str] = ...,
skip: int = ...,
) -> Sequence[_ManagedFileRow]: ...
async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
@ -1365,12 +1376,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_list(
self,
purpose: Optional[OpenAIFilesPurpose],
purpose: Optional[str],
litellm_parent_otel_span: Optional[Span],
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
) -> FileListPage:
"""List the managed files the caller owns, newest first.
Pagination is keyset based on ``unified_file_id`` so a key that owns
every file on the proxy still reads one bounded page at a time.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
``data`` non-empty while matches remain and its last id usable as the
next cursor. A first chunk that fills the page costs one query; once a
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, with no index
covering either the owner filter or the sort.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
owner_filter: Final = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return FileListPage(**build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
where={**owner_filter, "unified_file_id": after}
)
if cursor_row is None:
raise ProxyException(
message=f"Invalid 'after' cursor: no file found with id '{after}'.",
type="invalid_request_error",
param="after",
code=400,
openai_code="invalid_value",
)
page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
matches: Final[List[OpenAIFileObject]] = []
cursor_id = after
chunk_size = page_size + 1
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
)
if len(chunk) < chunk_size:
break
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
def _is_batch_polling_enabled(self) -> bool:
"""

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.58"
version = "0.1.59"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.58"
version = "0.1.59"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

4
litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql
View file

@ -1,4 +0,0 @@
UPDATE "LiteLLM_SpendLogs"
SET "created_at" = "endTime",
"updated_at" = "endTime"
WHERE "created_at" > "endTime" + interval '1 hour';

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.88"
version = "0.4.89"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.88"
version = "0.4.89"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

24
litellm/_logging.py
View file

@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str:
return _redact_string(value)
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.
uvicorn's colorized formatter re-renders `color_message` against
record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead
of using the already-formatted record.msg, so it has to be substituted
before args are cleared or it is later formatted with no args and prints
the raw "%s://%s:%d" placeholders instead of the URL.
"""
color_message: Final = record.__dict__.get("color_message")
if not isinstance(color_message, str) or not record.args:
return None
try:
return color_message % record.args
except TypeError:
return color_message
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter):
if not _ENABLE_SECRET_REDACTION:
return True
# Runs before args are cleared, and before the extra-field loop below
# that redacts the substituted result.
substituted_color_message: Final = _substituted_color_message(record)
if substituted_color_message is not None:
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
try:
record.msg = _redact_string(record.getMessage())
record.args = None

10
litellm/_redis.py
View file

@ -665,8 +665,16 @@ def get_redis_async_client(
cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
cluster_kwargs.setdefault("socket_keepalive", True)
# A single node's client-side timeout must reset only that node's connections,
# not tear down the whole cluster client for every concurrent caller.
from litellm.caching.redis_cluster_node_isolation import (
get_litellm_async_redis_cluster_class,
)
async_redis_cluster_class: Final = get_litellm_async_redis_cluster_class()
# Create async RedisCluster with IAM token as password if available
cluster_client: Final = async_redis.RedisCluster(
cluster_client: Final = async_redis_cluster_class(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)

40
litellm/batches/batch_utils.py
View file

@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage(
)
def _provider_output_file_id(output_file_id: str) -> str:
"""
Resolve the file id the provider actually knows: unified ids yield their embedded
llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_original_file_id,
)
unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id)
if not unified_file_id:
return get_original_file_id(output_file_id)
try:
extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s",
output_file_id,
e,
)
return output_file_id
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted)
return extracted
async def _fetch_batch_output_file_content(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content(
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
if batch.output_file_id is None:
raise ValueError("Output file id is None cannot retrieve file content")
file_id = batch.output_file_id
is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
try:
file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id)
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e
)
file_id: Final = _provider_output_file_id(batch.output_file_id)
# Build kwargs for afile_content with credentials from litellm_params
file_content_kwargs: Final = {

173
litellm/caching/redis_cluster_node_isolation.py Normal file
View file

@ -0,0 +1,173 @@
"""Bounds the blast radius of a single node's transient connection error on the async
Redis Cluster client.
redis-py's ``RedisCluster._execute_command`` responds to a ``ConnectionError`` or
``TimeoutError`` on ANY one node by tearing down every node's connections and flipping
the client into "needs reinitialization", which forces every other concurrent caller
sharing this client through one reinit lock until the whole cluster topology is
re-walked. Under real proxy load, a client-side socket timeout on a single node is a
routine event (the event loop was too busy to read the response before ``socket_timeout``
elapsed) and does not mean the cluster's topology moved, so treating it as a full-cluster
event turns one slow node into a proxy-wide latency spike while Redis itself stays
healthy -- confirmed live: pausing one of three local cluster nodes made every concurrent
command against the other two, untouched nodes stall for the full pause duration too.
``get_litellm_async_redis_cluster_class`` returns a ``RedisCluster`` subclass that resets
only the node that actually failed (mirroring what a plain, non-cluster Redis client
already does when one of its pooled connections errors), leaving every other node's
connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
topology changed.
"""
import asyncio
from typing import TYPE_CHECKING, Final, Protocol
from litellm._logging import verbose_logger
if TYPE_CHECKING:
from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
class _ClusterNodeAttrs(Protocol):
"""The subset of ``redis.asyncio.cluster.ClusterNode`` this override reads. redis-py
ships no resolvable stub for these members under the repo's current types-redis pin,
so a plain attribute access resolves every downstream use to ``Unknown`` under strict
mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
own logic fully typed without a banned ``typing.cast``."""
async def execute_command(
self,
*args: object,
**kwargs: object, # kwargs-ok: mirrors redis-py's own ClusterNode.execute_command signature, a raw command dispatch with no fixed keyword contract
) -> object: ...
async def disconnect(self) -> None: ...
class _NodesManagerAttrs(Protocol):
_moved_exception: object
def get_node_from_slot(
self, slot: int, read_from_replicas: bool, load_balancing_strategy: object
) -> _ClusterNodeAttrs: ...
class _ClusterAttrs(Protocol):
RedisClusterRequestTTL: int
reinitialize_counter: int
reinitialize_steps: int
read_from_replicas: bool
load_balancing_strategy: object
nodes_manager: _NodesManagerAttrs
def get_node(self, node_name: str) -> _ClusterNodeAttrs: ...
async def _determine_slot(self, *args: object) -> int: ...
async def aclose(self) -> None: ...
#: redis-py versions this override's copied ``_execute_command`` body has been verified
#: against. A version outside this set may have changed the method's structure in a way
#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
#: construction logs a loud warning rather than silently trusting an unverified copy.
_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
"""Builds the ``RedisCluster`` subclass with the per-node isolation fix.
Imported lazily because this module is reachable from a base ``import litellm`` while
redis is not a base dependency. Cheap to call repeatedly: the underlying redis
submodules are cached in ``sys.modules`` after the first import.
"""
import redis
from redis.asyncio.cluster import (
RedisCluster as _BaseAsyncRedisCluster, # pyright: ignore[reportUnknownVariableType] # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
)
from redis.cluster import get_node_name
from redis.commands import READ_COMMANDS
from redis.exceptions import (
AskError,
BusyLoadingError,
ClusterDownError,
ClusterError,
MaxConnectionsError,
MovedError,
SlotNotCoveredError,
TryAgainError,
)
from redis.exceptions import ConnectionError as _RedisConnectionError
from redis.exceptions import TimeoutError as _RedisTimeoutError
if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
verbose_logger.warning(
"redis-py %s is not in the set this cluster-teardown-storm fix was verified "
"against (%s). The per-node-isolation override may not match the installed library's "
"real _execute_command behavior.",
redis.__version__,
sorted(_VERIFIED_REDIS_VERSIONS),
)
class LiteLLMAsyncRedisCluster(
_BaseAsyncRedisCluster # pyright: ignore[reportUntypedBaseClass] # same stale-stub gap as the import above; the base class itself is unresolvable, not this subclass's own code
):
async def _execute_command(
self,
target_node: _ClusterNodeAttrs,
*args: object,
**kwargs: object, # kwargs-ok: overrides redis-py's own **kwargs signature; the keyword contract is defined by the Redis command being dispatched, not by this method
) -> object:
cluster: _ClusterAttrs = self
node = target_node
asking = moved = False
redirect_addr: str | None = None
ttl = cluster.RedisClusterRequestTTL
while ttl > 0:
ttl -= 1
try:
if asking:
assert redirect_addr is not None
node = cluster.get_node(node_name=redirect_addr)
await node.execute_command("ASKING")
asking = False
elif moved:
slot = await cluster._determine_slot(*args) # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch, which makes this identical private call from the same subclass
node = cluster.nodes_manager.get_node_from_slot(
slot,
cluster.read_from_replicas and args[0] in READ_COMMANDS,
(cluster.load_balancing_strategy if args[0] in READ_COMMANDS else None),
)
moved = False
return await node.execute_command(*args, **kwargs)
except (BusyLoadingError, MaxConnectionsError):
raise
except (_RedisConnectionError, _RedisTimeoutError):
# Reset only the node that actually failed instead of the upstream
# default (`await self.aclose()`, a full-cluster teardown that forces
# every other concurrent caller through the shared reinit lock).
await node.disconnect()
raise
except (ClusterDownError, SlotNotCoveredError):
await cluster.aclose()
await asyncio.sleep(0.25)
raise
except MovedError as e:
cluster.reinitialize_counter += 1
if cluster.reinitialize_steps and cluster.reinitialize_counter % cluster.reinitialize_steps == 0:
await cluster.aclose()
cluster.reinitialize_counter = 0
else:
cluster.nodes_manager._moved_exception = e # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch; redis-py exposes no public setter for this
moved = True
except AskError as e:
redirect_addr = get_node_name(host=e.host, port=e.port)
asking = True
except TryAgainError:
if ttl < cluster.RedisClusterRequestTTL / 2:
await asyncio.sleep(0.05)
raise ClusterError("TTL exhausted.")
return LiteLLMAsyncRedisCluster

7
litellm/constants.py
View file

@ -783,6 +783,7 @@ openai_compatible_endpoints: Final[list] = [
"https://pinstripes.io/v1",
"https://api.meta.ai/v1",
"https://api.cognition.ai/v1",
"https://api.scx.ai/v1",
]
@ -851,6 +852,7 @@ openai_compatible_providers: Final[list] = [
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
"scx-ai",
]
openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions`
"together_ai",
@ -1354,6 +1356,8 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@ -1534,9 +1538,12 @@ TOOL_SPEND_TOP_TOOLS: Final = 100
SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3")))
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2"))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))

14
litellm/integrations/callback_configs.json
View file

@ -294,12 +294,18 @@
"id": "newrelic",
"displayName": "New Relic",
"logo": "newrelic.png",
"supports_key_team_logging": false,
"supports_key_team_logging": true,
"dynamic_params": {
"NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
"newrelic_api_key": {
"type": "password",
"ui_name": "New Relic Ingest License Key",
"description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
"required": false
},
"newrelic_region": {
"type": "text",
"ui_name": "Record AI Content (default: true)",
"description": "Whether to record AI message content. Set to false to disable.",
"ui_name": "New Relic Region (us or eu)",
"description": "Data center region for this team's account. Defaults to us.",
"required": false
}
},

19
litellm/integrations/custom_logger.py
View file

@ -60,6 +60,25 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
# Class variables or attributes
enforces_request_content: bool = False
"""
Whether this hook's ``async_pre_call_hook`` judges the request payload itself.
False for the accounting hooks, which count a request rather than read it: rate limits,
parallel slots, budgets, cache lookups. Those must run once per request and never once per
record of a batch upload, which would charge a caller once for every line of their file.
Set it to True on a hook that inspects or rejects content, so that scanning a payload which
is not itself a request, such as one record of a batch input file, still reaches it. A
``CustomGuardrail`` does not need it; guardrails are dispatched by their own branch.
Judging content is necessary but not sufficient. A hook that also rewrites the payload for
routing, as the managed-files and managed-vector-store hooks do, stays False: a per-record
rewrite would read as a redaction and ship embedded in the record. Only the leaf class is
consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
"""
def __init__(
self,
turn_off_message_logging: bool = False,

4
litellm/integrations/otel/__init__.py
View file

@ -49,6 +49,7 @@ from litellm.integrations.otel.model.semconv import (
Error,
GenAI,
GenAIOperation,
GenAIOutputType,
GenAIProvider,
JsonRpc,
LiteLLM,
@ -60,6 +61,7 @@ from litellm.integrations.otel.model.semconv import (
RpcSystem,
Server,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.spans import (
@ -84,6 +86,7 @@ __all__ = [
"Error",
"GenAI",
"GenAIOperation",
"GenAIOutputType",
"GenAIProvider",
"GuardrailSpanData",
"JsonRpc",
@ -116,6 +119,7 @@ __all__ = [
"is_otel_v2_enabled",
"promoted_baggage",
"resolve_operation",
"resolve_output_type",
"resolve_provider",
"span_role_for_service",
"validate_registry",

6
litellm/integrations/otel/emitter.py
View file

@ -156,6 +156,12 @@ class SpanEmitter:
links=list(links) if links else None,
)
def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
"""Register a span emitted outside :meth:`emit` (the boundary-opened
LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
for the same ``(dedup_key, role)`` deduplicates against it."""
self._seen(dedup_key, role)
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
"""Return True once a ``(dedup_key, role)`` pair has been emitted.

36
litellm/integrations/otel/logger.py
View file

@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
if carrier is None:
# A missing carrier does not always mean nothing happened: a team/key-scoped
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
return None
try:
return self._finish_carrier(carrier, call, end_time)
return self._finish_carrier(carrier, call, start_time, end_time)
finally:
# After the span has ended, so a release-triggered provider shutdown
# force-flushes it out rather than racing its enqueue.
@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
"""Remember an in-flight LLM call, evicting the oldest if over budget.
A call that opens but never closes (a stream that only fires stream
events) would linger otherwise; the evicted span is simply dropped
(never exported).
events) would linger otherwise. Eviction only drops the boundary carrier,
not the call: if that call later closes as a real completed call, it still
emits through the deferred branch in ``_close_llm_call`` (the same path a
team/key-scoped logger uses, since it never opens a carrier), deduplicated
by call id. Only a call that is evicted and never closes goes unexported.
"""
self._open_llm_calls[call_id] = carrier
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
def _finish_carrier(
self,
carrier: _LLMCallSpan,
carrier: "_LLMCallSpan | None",
call: LLMCallEvent,
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
payload: Final = call.payload
call_id: Final = call.call_id
if payload is None:
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Opened at the boundary but the payload never materialized — end
# it (named provisionally) so it isn't leaked as an open span.
# it (named provisionally) so it isn't leaked as an open span, and
# register the dedup marker so a later payload-carrying close for
# the same call id cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
carrier.span.end(end_time=to_ns(end_time))
return None
data: Final = LLMCallSpanData.from_standard_logging_payload(
@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
)
end_time_ns: Final = to_ns(end_time)
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set
# status, and end it. Its parent (the server span) was captured at
# creation from real ambient context.
# creation from real ambient context. Register the dedup marker so a
# second close for the same call id (success then failure on one
# logging object) cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
return carrier.span
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
SpanRole.LLM_CALL,
data,
parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
start_time_ns=carrier.start_time_ns,
start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
end_time_ns=end_time_ns,
tracer=route.tracer,
links=_request_trace_links(parent_ctx) if route.detached else None,

2
litellm/integrations/otel/mappers/genai.py
View file

@ -42,6 +42,7 @@ class GenAIMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
GenAI.OPERATION_NAME: lambda d: d.operation.value,
GenAI.PROVIDER_NAME: lambda d: d.provider or None,
GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
@ -65,6 +66,7 @@ class GenAIMapper:
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
LiteLLM.CALL_TYPE: lambda d: d.call_type,
# The provider/underlying model is only known once routing has picked a
# deployment, so it can't ride identity Baggage (seeded at auth, before
# routing) onto the boundary-born LLM span — stamp it directly here.

10
litellm/integrations/otel/model/config.py
View file

@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
WEAVE_OTEL = "weave_otel"
LEVO = "levo"
AGENTOPS = "agentops"
NEWRELIC = "newrelic"
class _OTelV2Flag(BaseSettings):
@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
"auto (Simple for console/in_memory, Batch otherwise)."
),
)
requires_headers: bool = Field(
default=False,
description=(
"Skip this exporter when no headers are resolved. For destinations "
"that reject unauthenticated exports (e.g. New Relic), a spec kept "
"only as the per-request credential-stamping target would otherwise "
"export keyless traffic and produce a 4xx for every span batch."
),
)
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -203,6 +203,11 @@ class LLMCallEvent:
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# True once the request handed off to a provider (``pre_call`` stamped
# ``api_call_start_time``). The affirmative signal that an LLM call was
# actually attempted — router pre-call rejections, SDK failures before the
# provider handoff, and standalone guardrail runs all lack it.
upstream_started: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -221,6 +226,7 @@ class LLMCallEvent:
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
auth_metadata=auth_metadata(payload, kwargs),
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
upstream_started=kwargs.get("api_call_start_time") is not None,
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
)

12
litellm/integrations/otel/model/payloads.py
View file

@ -15,8 +15,10 @@ from litellm.integrations.otel.model.metadata import (
)
from litellm.integrations.otel.model.semconv import (
GenAIOperation,
GenAIOutputType,
MCPMethod,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.utils import (
@ -310,6 +312,11 @@ class LLMCallSpanData:
choices_out: tuple[Mapping[str, object], ...] = ()
system_fingerprint: str | None = None
time_to_first_chunk_seconds: float | None = None
# The requested output modality, set only on the routes that pin one (image
# generation, speech, transcription, OCR), and the litellm route itself, which
# keeps routes the convention folds into one operation distinguishable.
output_type: GenAIOutputType | None = None
call_type: str | None = None
@classmethod
def from_standard_logging_payload(
@ -334,8 +341,9 @@ class LLMCallSpanData:
# otherwise the content-bearing mappers receive empty sequences and emit
# no prompt/response text.
finish_reasons: Final = _finish_reasons(choices_out)
call_type: Final = as_str(payload.get("call_type"))
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
operation=resolve_operation(call_type),
provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
request_model=context.request_model,
response_model=context.response_model,
@ -358,6 +366,8 @@ class LLMCallSpanData:
choices_out=choices_out if capture_content else (),
system_fingerprint=as_str(response.get("system_fingerprint")),
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
output_type=resolve_output_type(call_type),
call_type=call_type or None,
)

57
litellm/integrations/otel/model/semconv.py
View file

@ -3,7 +3,9 @@ Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anythin
without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
"""
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
@ -30,6 +32,21 @@ class GenAIOperation(str, Enum):
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_MODERATION = "litellm.moderation"
class GenAIOutputType(str, Enum):
"""Values for ``gen_ai.output.type``, the modality the client asked for.
It is what separates the inference routes that share ``generate_content``:
image generation requests ``image``, speech requests ``speech``, and
transcription and OCR both request ``text``.
"""
TEXT = "text"
JSON = "json"
IMAGE = "image"
SPEECH = "speech"
class GenAIProvider(str, Enum):
@ -258,6 +275,11 @@ class LiteLLM:
"""Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
CALL_ID: Final = "litellm.call_id"
# The litellm route that produced the call. Needed because the convention maps
# several routes onto one operation: transcription and OCR are both
# ``generate_content`` with a ``text`` output type, so this is the only thing
# that tells them apart.
CALL_TYPE: Final = "litellm.call_type"
COST_PREFIX: Final = "litellm.cost."
METADATA_PREFIX: Final = "litellm.metadata."
TEAM_ID: Final = "litellm.team.id"
@ -352,6 +374,16 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"image_generation": GenAIOperation.GENERATE_CONTENT,
"aimage_generation": GenAIOperation.GENERATE_CONTENT,
"moderation": GenAIOperation.LITELLM_MODERATION,
"amoderation": GenAIOperation.LITELLM_MODERATION,
"ocr": GenAIOperation.GENERATE_CONTENT,
"aocr": GenAIOperation.GENERATE_CONTENT,
"speech": GenAIOperation.GENERATE_CONTENT,
"aspeech": GenAIOperation.GENERATE_CONTENT,
"transcription": GenAIOperation.GENERATE_CONTENT,
"atranscription": GenAIOperation.GENERATE_CONTENT,
"call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
"vector_store_search": GenAIOperation.RETRIEVAL,
"avector_store_search": GenAIOperation.RETRIEVAL,
@ -385,6 +417,23 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
}
# litellm ``call_type`` -> ``gen_ai.output.type``. Only the call types whose route
# fixes the requested modality are listed; the attribute is conditionally required
# on a request that asks for an output format, so anything else is left unstamped.
_OUTPUT_TYPE_BY_CALL_TYPE: Final[Mapping[str, GenAIOutputType]] = MappingProxyType(
{
"image_generation": GenAIOutputType.IMAGE,
"aimage_generation": GenAIOutputType.IMAGE,
"speech": GenAIOutputType.SPEECH,
"aspeech": GenAIOutputType.SPEECH,
"transcription": GenAIOutputType.TEXT,
"atranscription": GenAIOutputType.TEXT,
"ocr": GenAIOutputType.TEXT,
"aocr": GenAIOutputType.TEXT,
}
)
def resolve_provider(custom_llm_provider: str | None) -> str:
"""Map a litellm provider string to a ``gen_ai.provider.name`` value.
@ -416,3 +465,11 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
GenAIOperation.CHAT.value,
)
return GenAIOperation.CHAT
def resolve_output_type(call_type: str | None) -> GenAIOutputType | None:
"""Map a litellm ``call_type`` to a ``gen_ai.output.type`` value, or ``None``
for a route that doesn't pin the output modality."""
if not call_type:
return None
return _OUTPUT_TYPE_BY_CALL_TYPE.get(call_type.lower())

2
litellm/integrations/otel/plumbing/providers.py
View file

@ -436,6 +436,8 @@ def build_tracer_provider(
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
exp = _exporter_from_spec(spec)
provider.add_span_processor(
_processor_for(

29
litellm/integrations/otel/plumbing/routing.py
View file

@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import (
get_tracer,
)
from litellm.integrations.otel.presets import (
dynamic_otlp_endpoint,
dynamic_otlp_headers,
project_routing_headers,
)
@ -129,7 +130,9 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
# Oldest-first so an overflow of draining providers sheds the stalest.
self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers
@ -182,12 +185,16 @@ class TenantTracerCache:
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
cache_key: Final = (
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -200,15 +207,16 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems],
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
self._providers[cache_key] = built
return built
@ -257,6 +265,7 @@ class TenantTracerCache:
self,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -272,7 +281,8 @@ class TenantTracerCache:
``Authorization``), which must survive routing to a project.
"""
exporters: Final = [
self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
@ -281,6 +291,7 @@ class TenantTracerCache:
spec: ExporterSpec,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> ExporterSpec:
kind: Final = spec.kind.lower()
if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@ -291,4 +302,10 @@ class TenantTracerCache:
if project_headers and kind not in _GRPC_KINDS
else base
)
return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
update: Final = { # mutable-ok: model_copy(update=...) requires a plain dict
field: value
for field, value in (("headers", routed), ("endpoint", endpoint))
if (field == "headers" and routed != spec.headers)
or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
}
return spec if not update else spec.model_copy(update=update)

70
litellm/integrations/otel/presets/__init__.py
View file

@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
)
from litellm.integrations.otel.presets.langtrace import langtrace_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.newrelic import (
newrelic_dynamic_endpoint,
newrelic_dynamic_headers,
newrelic_preset,
)
from litellm.integrations.otel.presets.phoenix import (
phoenix_preset,
phoenix_project_headers,
@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
#: registered value matches the preset interface.
PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"weave_otel": weave_preset,
}
PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
{
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"newrelic": newrelic_preset,
"weave_otel": weave_preset,
}
)
#: Callback name → per-request OTLP header builder (team/key multi-tenant
#: routing). Only integrations that support dynamic credentials appear here —
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
#: default tracer.
DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
MappingProxyType(
{
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"newrelic": newrelic_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
)
)
#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
#: destination host varies per tenant (from a fixed region table, never a
#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
#: authoritative.
DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
MappingProxyType(
{
"newrelic": newrelic_dynamic_endpoint,
}
)
)
#: Callback name → per-request *routing* header builder, sourced from the key/team
@ -98,17 +123,34 @@ def project_routing_headers(
return builder(auth_metadata)
def dynamic_otlp_endpoint(
callback_name: str | None,
dynamic_params: StandardCallbackDynamicParams | None,
) -> str | None:
"""Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
``None`` means "keep the preset's own endpoint".
"""
resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
if resolver is None or not dynamic_params:
return None
return resolver(dynamic_params)
__all__ = [
"DYNAMIC_ENDPOINT_BY_CALLBACK",
"DYNAMIC_HEADERS_BY_CALLBACK",
"PRESET_BY_CALLBACK",
"PROJECT_HEADERS_BY_CALLBACK",
"Preset",
"agentops_preset",
"arize_preset",
"dynamic_otlp_endpoint",
"dynamic_otlp_headers",
"langfuse_preset",
"langtrace_preset",
"levo_preset",
"newrelic_preset",
"phoenix_preset",
"project_routing_headers",
"weave_preset",

104
litellm/integrations/otel/presets/newrelic.py Normal file
View file

@ -0,0 +1,104 @@
"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import (
ExporterOwner,
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
#: region enum rather than a free-form endpoint, so callback vars can never
#: redirect telemetry to an arbitrary host.
NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
{
"us": "https://otlp.nr-data.net",
"eu": "https://otlp.eu01.nr-data.net",
}
)
_DEFAULT_REGION: Final = "us"
class _NewRelicSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
# The same env vars the agent-based integration documents; the key is the
# operator-level fallback for traffic without team credentials, the region
# picks that fallback's data center, and the record-content flag keeps its
# documented meaning when the OTel path replaces the agent.
license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
def newrelic_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
settings: Final = _NewRelicSettings()
base: Final = config_overrides or OpenTelemetryV2Config()
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
(settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
)
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,
headers=(f"api-key={settings.license_key}" if settings.license_key else None),
owner=ExporterOwner.NEWRELIC,
requires_headers=True,
),
],
# New Relic ingests the OTLP GenAI semantic conventions natively.
"mapper_names": ensure_mappers(base.mapper_names, "genai"),
**(
{"capture_message_content": ("span_only" if settings.record_content else "no_content")}
if settings.record_content is not None
else {}
),
}
)
def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request New Relic OTLP headers from team/key dynamic params."""
api_key: Final = params.get("newrelic_api_key")
return {header: value for header, value in (("api-key", api_key),) if value}
def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
"""Per-request OTLP endpoint for the team's ``newrelic_region``.
Always the team's own region endpoint, defaulting to US when the team left
the region unset. It never falls through to the preset's endpoint, which
follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
ingest key must not inherit the operator's region and have its US-account
spans rejected by an EU-configured default (or vice versa). An unknown
region likewise resolves to the documented US default rather than a guess.
"""
region: Final = params.get("newrelic_region")
default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
if not region:
return default_endpoint
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
region,
", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
)
return default_endpoint
return endpoint

27
litellm/integrations/prometheus.py
View file

@ -215,7 +215,9 @@ class PrometheusLogger(CustomLogger):
# request latency metrics
self.litellm_request_total_latency_metric = self._histogram_factory(
"litellm_request_total_latency_metric",
"Total latency (seconds) for a request to LiteLLM",
"End-to-end latency (seconds) for a request to LiteLLM Proxy Server, from the moment "
"the request reached the proxy through the end of processing -- includes "
"authentication, pre-call hooks, the LLM API call, and post-call processing",
labelnames=self.get_labels_for_metric("litellm_request_total_latency_metric"),
buckets=self.latency_buckets,
)
@ -458,7 +460,8 @@ class PrometheusLogger(CustomLogger):
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time spent in request queue before processing starts (seconds)",
"Time (seconds) from request arrival at the proxy to the start of pre-call "
"processing -- includes authentication and any ASGI-level queueing",
labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
buckets=self.latency_buckets,
)
@ -2078,27 +2081,37 @@ class PrometheusLogger(CustomLogger):
_labels,
)
# total request latency
# request queue time (time from arrival to processing start) -- read first so
# it can be folded into the total-latency metric below. start_time/end_time
# only span from after auth completes, so without this the "total" latency
# metric silently excludes auth and pre-call hook time.
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
# total request latency: true end-to-end, from request arrival (queue_time_seconds,
# when available) through the end of processing.
total_time_seconds: Final = self._safe_duration_seconds(
start_time=start_time,
end_time=end_time,
)
if total_time_seconds is not None:
_observed_total_time_seconds: Final = (
total_time_seconds + queue_time_seconds
if queue_time_seconds is not None and queue_time_seconds >= 0
else total_time_seconds
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_request_total_latency_metric.labels(**_labels).observe(total_time_seconds)
self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
if queue_time_seconds is not None and queue_time_seconds >= 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_queue_time_seconds"),

53
litellm/integrations/websearch_interception/ARCHITECTURE.md
View file

@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
---
## Loop Ceiling
One intercepted request can chain several follow-up model calls, since the model often searches again after
reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
```yaml
litellm_settings:
websearch_interception_params:
enabled_providers: ["bedrock"]
max_agentic_loops: 5
```
Or per deployment, which wins over the feature-level setting:
```yaml
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
max_agentic_loops: 5
```
Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
carries it is ignored and one request can never drive an unbounded number of upstream model calls.
Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
validator down there would drop the deployment silently instead of refusing to start.
When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
rather than as a replacement.
Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
Both are tracked separately
---
## Streaming Support
WebSearch interception works transparently with both streaming and non-streaming requests.

22
litellm/integrations/websearch_interception/handler.py
View file

@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
self,
enabled_providers: list[LlmProviders | str] | None = None,
search_tool_name: str | None = None,
max_agentic_loops: int | None = None,
):
"""
Args:
@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
Default: None (all providers enabled)
search_tool_name: Name of search tool configured in router's search_tools.
If None, will attempt to use first available search tool.
max_agentic_loops: How many follow-up model calls one intercepted request
may chain before the loop is refused and the turn ends.
If None, LiteLLM's default of 3 applies.
"""
super().__init__()
# Convert enum values to strings for comparison
@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
else:
self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
self.search_tool_name = search_tool_name
self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
self._request_has_websearch = False # Track if current request has web search
@staticmethod
def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
"""
Reject loop ceilings the agentic loop cannot honor, at config load time.
"""
return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
async def try_short_circuit_search(
self,
model: str,
@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: "my-perplexity-search"
max_agentic_loops: 5
Usage:
config = litellm_settings.get("websearch_interception_params", {})
@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# Extract parameters from config
enabled_providers_str: Final = config.get("enabled_providers", None)
search_tool_name: Final = config.get("search_tool_name", None)
max_agentic_loops: Final = config.get("max_agentic_loops", None)
# Convert string provider names to LlmProviders enum values
enabled_providers: list[LlmProviders | str] | None = None
@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return cls(
enabled_providers=enabled_providers,
search_tool_name=search_tool_name,
max_agentic_loops=max_agentic_loops,
)
@staticmethod
@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag

59
litellm/litellm_core_utils/agentic_loop_settings.py Normal file
View file

@ -0,0 +1,59 @@
"""
Shared validation for the agentic loop ceiling.
``max_agentic_loops`` can be set in two places, and the two disagreed about
what a bad value means. The feature-level
``litellm_settings.websearch_interception_params.max_agentic_loops`` was
checked at config load, while a per-deployment
``model_list[].litellm_params.max_agentic_loops`` was passed straight through
to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
turning the tightest ceiling into the loosest one, and let a per-deployment
``"three"`` boot the proxy and then fail every request to that model.
Both settings now go through :func:`validated_max_agentic_loops`, which names
the field it rejected so the error says which line of the config to fix.
Anything that spells a whole number is still accepted, because the old
``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
"""
from typing import Final
DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
def _as_whole_number(value: object) -> int | None:
"""
Return ``value`` as an int when it spells a whole number, else ``None``.
``bool`` is excluded explicitly because it is an ``int`` subclass, so
``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
"""
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value) if value.is_integer() else None
if isinstance(value, str):
try:
return int(value.strip())
except ValueError:
return None
return None
def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
"""
Return ``max_agentic_loops`` as an int, or raise naming ``field``.
"""
if max_agentic_loops is None:
return None
ceiling: Final = _as_whole_number(max_agentic_loops)
if ceiling is None:
raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
if ceiling < 1:
raise ValueError(f"{field} must be at least 1, got {ceiling}")
return ceiling

9
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -5,6 +5,10 @@ from typing import Final, cast
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
return depth, max_loops, fingerprints

28
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -811,6 +811,24 @@ def _map_openai_like_exception(
)
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
"""
Mantle reports context overflow as a structured validation error rather than
the plain-text patterns Bedrock itself uses, so it needs its own detection and a
message clients recognize as context overflow (litellm/litellm#36546).
"""
if "invalid_request_error" not in error_str and "validation_error" not in error_str:
return None
match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
if match is None:
return None
prompt_tokens, max_tokens = match.groups()
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
def _map_bedrock_exception(
*,
model: str,
@ -821,6 +839,14 @@ def _map_bedrock_exception(
exception_provider: str,
extra_information: str,
) -> None:
if custom_llm_provider == "bedrock_mantle":
mantle_context_window_message = _get_bedrock_mantle_context_window_message(error_str)
if mantle_context_window_message is not None:
raise ContextWindowExceededError(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
)
if (
"too many tokens" in error_str
or "expected maxLength:" in error_str
@ -2315,7 +2341,7 @@ def exception_type(
exception_provider=exception_provider,
extra_information=extra_information,
)
elif custom_llm_provider == "bedrock":
elif custom_llm_provider in ("bedrock", "bedrock_mantle"):
_map_bedrock_exception(
model=model,
original_exception=mappable_exception,

24
litellm/litellm_core_utils/get_provider_specific_headers.py
View file

@ -1,3 +1,4 @@
from collections.abc import Sequence
from typing import Final
from litellm.types.utils import ProviderSpecificHeader
@ -6,13 +7,17 @@ from litellm.types.utils import ProviderSpecificHeader
class ProviderSpecificHeaderUtils:
@staticmethod
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | None,
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
custom_llm_provider: str | None,
) -> dict:
"""
Get the provider specific headers for the given custom llm provider.
Supports comma-separated provider lists for headers that work across multiple providers.
Accepts either a single ProviderSpecificHeader or a sequence of them. Each entry
carries its own comma-separated provider list, so headers that are safe for several
providers and headers that are safe for exactly one can travel on the same request
without sharing a scope. Entries whose provider list does not contain
`custom_llm_provider` contribute nothing.
Returns:
Dict: The provider specific headers for the given custom llm provider
@ -20,10 +25,15 @@ class ProviderSpecificHeaderUtils:
if provider_specific_header is None or custom_llm_provider is None:
return {}
stored_providers: Final = provider_specific_header.get("custom_llm_provider", "")
provider_list: Final = [p.strip() for p in stored_providers.split(",")]
scoped_headers: Final = (
(provider_specific_header,) if isinstance(provider_specific_header, dict) else provider_specific_header
)
if custom_llm_provider in provider_list:
return provider_specific_header.get("extra_headers", {})
matched_headers: Final = {}
for scoped_header in scoped_headers:
stored_providers = scoped_header.get("custom_llm_provider", "")
provider_list = [p.strip() for p in stored_providers.split(",")]
if custom_llm_provider in provider_list:
matched_headers.update(scoped_header.get("extra_headers", {}))
return {}
return matched_headers

32
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final = [
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
@ -72,8 +72,10 @@ _supported_callback_params: Final = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
"turn_off_message_logging",
]
)
_request_blocked_callback_params: Final = frozenset(
{
@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset(
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
}
)
# Request-blocked params that must still reach ``standard_callback_dynamic_params``
# when the proxy itself stamped them from admin-configured team/key callback
# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads
# ``standard_callback_dynamic_params``, so without this overlay a blocked param
# could never drive routing at all.
_trusted_overlay_callback_params: Final = frozenset(
{
"newrelic_api_key",
"newrelic_region",
}
)
@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params(
if param in kwargs:
_param_value = kwargs.get(param)
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params(
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for param, trusted_value in get_trusted_callback_params(kwargs):
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
return standard_callback_dynamic_params

52
litellm/litellm_core_utils/litellm_logging.py
View file

@ -4503,6 +4503,9 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
for callback in _in_memory_loggers:
if isinstance(callback, NewRelicLogger):
return callback
@ -4789,7 +4792,11 @@ def get_custom_logger_compatible_class(
if isinstance(callback, SMTPEmailLogger):
return callback
elif logging_integration == "newrelic":
from litellm.integrations.otel.logger import OpenTelemetryV2
for callback in _in_memory_loggers:
if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic":
return callback
if isinstance(callback, NewRelicLogger):
return callback
return None
@ -5615,6 +5622,37 @@ def _extract_response_obj_and_hidden_params(
return response_obj, hidden_params
def _autorouter_savings_for_payload(
request_metadata: Mapping[str, object],
model: str | None,
custom_llm_provider: str | None,
model_id: str | None,
usage_object: Mapping[str, object] | None,
cost_breakdown: Mapping[str, object] | None,
) -> float | None:
"""The auto-router savings figure for the payload, or ``None`` when there is none.
Lazy proxy import: the savings module lives with the spend trackers that own the
math, and SDK-only installs have no proxy package to import.
"""
try:
from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload
except Exception: # noqa: BLE001 # SDK-only install: no savings driver to run
return None
try:
return autorouter_savings_for_logging_payload(
request_metadata=request_metadata,
model=model,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
usage_object=usage_object,
cost_breakdown=cost_breakdown,
)
except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging
verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
return None
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: Any | BaseModel | dict,
@ -5772,6 +5810,16 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
autorouter_savings: Final = _autorouter_savings_for_payload(
request_metadata=metadata,
model=model_name,
custom_llm_provider=custom_llm_provider,
model_id=_model_id,
usage_object=usage_dict,
cost_breakdown=request_cost_breakdown,
)
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
id=str(id),
litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
@ -5802,7 +5850,8 @@ def get_standard_logging_object_payload(
metadata=clean_metadata,
cache_key=clean_hidden_params["cache_key"],
response_cost=response_cost,
cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,
total_tokens=usage_dict.get("total_tokens", 0),
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
@ -5998,6 +6047,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
call_type="completion",
stream=False,
response_cost=response_cost,
autorouter_savings=None,
response_cost_failure_debug_info=None,
status="success",
total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT),

11
litellm/litellm_core_utils/logging_utils.py
View file

@ -3,6 +3,7 @@ import functools
import inspect
import re
import time
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
@ -268,6 +269,16 @@ def _set_duration_in_model_call_details(
verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e)
def speech_request_body(model: str, voice: str, optional_params: Mapping[str, object]) -> Mapping[str, object]:
"""Speech request body for telemetry, without the caller headers the provider SDKs
take as request kwargs rather than body fields."""
return { # mutable-ok: loggers isinstance-check the request body as a dict
"model": model,
"voice": voice,
**{key: value for key, value in optional_params.items() if key != "extra_headers"},
}
def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.

52
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -8,7 +8,7 @@ router prices at zero serves its traffic for free.
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import date, datetime, time, timezone
from types import MappingProxyType
from typing import Final
@ -68,9 +68,17 @@ def _to_utc(parsed: datetime) -> datetime:
def _as_utc(value: object) -> datetime | None:
"""A model_info datetime as UTC, parsing an ISO string, else None."""
"""A model_info datetime as UTC, parsing an ISO string, else None.
An unquoted ``2027-01-01`` in config.yaml is loaded as a ``date``, not a string, and a
reservation bound that fails to parse takes the whole deployment out of PTU handling,
so the day is read as its opening midnight rather than discarded. ``datetime`` derives
from ``date``, so it has to be matched first.
"""
if isinstance(value, datetime):
return _to_utc(value)
if isinstance(value, date):
return datetime.combine(value, time.min, tzinfo=timezone.utc)
if not isinstance(value, str):
return None
try:
@ -84,6 +92,46 @@ def _named(reason: str, model_name: str | None) -> str:
return reason if model_name is None else f"PTU configuration on model '{model_name}' is invalid: {reason}"
def ptu_identity_error(
*, declared_id: str | None, taken: bool, current_id: str | None = None, model_name: str | None = None
) -> str | None:
"""Why this config-declared reservation cannot be identified, else None.
A deployment declared in config.yaml is otherwise keyed by a hash of its resolved
``litellm_params``, so rotating a credential or editing an endpoint mints a second
identity and the reservation is charged again under it. The flat cost is keyed by that
id, and a charge already written is never retracted, so the duplicate is permanent.
``current_id`` is what the deployment is keyed by today. Naming it is the difference
between an operator carrying their history forward and an operator inventing a fresh
id, which starts a second identity beside the charges already written.
"""
if not declared_id:
return _named(
"model_info.id is required when PTU fields are set. Without one the deployment is "
"identified by a hash of its litellm_params, so rotating a credential bills the "
"reservation a second time under the new identity. Set it to the id this deployment "
f"already uses, {current_id or 'shown by GET /model/info'}, so the flat cost already "
"written stays under one identity; any other value starts a second one",
model_name,
)
if taken:
return _named(
f"model_info.id '{declared_id}' is declared on more than one deployment. Each would key "
"the same flat-cost row, so one reservation would go unbilled",
model_name,
)
return None
PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to")
def declares_ptu(model_info: Mapping[str, object]) -> bool:
"""Whether any PTU field is set here, including one too malformed to charge."""
return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS)
def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
"""Why this PTU configuration cannot be honoured, else None.

6
litellm/llms/anthropic/chat/transformation.py
View file

@ -1827,6 +1827,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
custom_llm_provider=self.custom_llm_provider,
)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
optional_params=optional_params,
custom_llm_provider=self._resolved_provider,
)
headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
# === Tool-name sanitization (single chokepoint) ===

35
litellm/llms/anthropic/common_utils.py
View file

@ -32,6 +32,12 @@ from litellm.types.llms.anthropic import (
from litellm.types.llms.openai import AllMessageValues
from litellm.types.proxy.model_listing import ModelInfoResponse
DROP_DISABLED_THINKING_WARNING: Final = (
"Dropping `thinking={'type': 'disabled'}` for model=%s: thinking is always on for this model and cannot be "
"disabled (the alternative is a provider 400). The model will still think adaptively, its response can contain "
"thinking blocks, and those thinking tokens are billed as output tokens."
)
_BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
_INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
_DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
@ -425,6 +431,35 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider)
@staticmethod
def _is_always_on_thinking_model(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` always thinks and rejects ``thinking.type=disabled``
(Fable 5 / Mythos 5 generation). The model cost map is authoritative: an
explicit ``thinking_always_on`` entry resolved under ``custom_llm_provider``,
or a ``fallback_generalizations`` rule for unmapped ids of those families.
"""
return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
@staticmethod
def maybe_drop_disabled_thinking(
model: str,
optional_params: dict, # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
custom_llm_provider: str,
) -> None:
"""Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models
(Fable 5 / Mythos 5), which 400 on it; omission is the API-documented
remedy and yields the model's default adaptive thinking."""
thinking: Final = optional_params.get("thinking")
if not isinstance(thinking, dict) or thinking.get("type") != "disabled":
return
if not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider):
return
litellm.verbose_logger.warning(
DROP_DISABLED_THINKING_WARNING,
model,
)
optional_params.pop("thinking", None)
def is_effort_used(
self,
optional_params: dict | None,

11
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
)
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
local_model_name,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
except Exception:
pass
if isinstance(model, str) and model and not model.startswith("responses/"):
# Prefix model with "responses/" to route to OpenAI Responses API
completion_kwargs["model"] = f"responses/{model}"
if isinstance(model, str) and model and "responses/" not in model:
local_model: Final = model.removeprefix(f"{custom_llm_provider}/")
completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}"
auto_summary: Final = is_reasoning_auto_summary_enabled()
@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=True,
@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=False,

32
litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py
View file

@ -56,7 +56,7 @@ from ..result import PolyfillResult
# so the summary's spend is attributed to the same scopes. The list mirrors the
# fields populated by
# ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata``.
# ``user_api_key_model_max_budget`` / ``user_api_key_end_user_model_max_budget``
# The three ``*_model_max_budget`` fields
# are what ``_PROXY_VirtualKeyModelMaxBudgetLimiter`` reads post-call to update
# the per-model spend caches, so without them the summary spend would never
# count against the caller's model budget. ``user_api_key_end_user_id`` /
@ -76,6 +76,7 @@ _PROPAGATED_METADATA_KEYS: Final = (
"user_api_key_end_user_id",
"user_api_end_user_max_budget",
"user_api_key_model_max_budget",
"user_api_key_user_model_max_budget",
"user_api_key_end_user_model_max_budget",
"litellm_call_id",
"litellm_parent_otel_span",
@ -317,10 +318,14 @@ async def _check_summary_model_budget(
The summary subrequest never passes back through ``user_api_key_auth``, so
without this gate a caller whose ``model_max_budget`` for
``context_management_summary_model`` is exhausted could keep consuming that
model via compaction. Mirrors the ``model_max_budget`` /
``end_user_model_max_budget`` enforcement that ``user_api_key_auth`` runs for
the client-requested model. Returns True outside the proxy or when no
model via compaction. Mirrors the per-model budget enforcement that
``user_api_key_auth`` runs for the client-requested model. Returns True outside the proxy or when no
per-model budget is configured.
All three scopes are checked because the summary's spend is charged to all
three: this file propagates the key, user and end-user budgets into the
subrequest's metadata, so enforcing only two of them would let compaction
increment a counter it can never be refused by.
"""
if user_api_key_auth is None:
return True
@ -347,6 +352,25 @@ async def _check_summary_model_budget(
)
return False
user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
user_id: Final = getattr(user_api_key_auth, "user_id", None)
if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
try:
await model_max_budget_limiter.is_user_within_model_budget(
user_id=user_id,
user_model_max_budget=user_model_max_budget,
model=summary_model,
)
except litellm.BudgetExceededError:
return False
except Exception as e: # noqa: BLE001 # a budget gate denies on any failure, as the key and end-user scopes do
verbose_logger.warning(
"compact_20260112: unexpected error during user model-budget check for summary_model=%s; denying: %s",
summary_model,
e,
)
return False
end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None)
end_user_id: Final = getattr(user_api_key_auth, "end_user_id", None)
if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None:

8
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
View file

@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
else:
passthrough_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": block_dict,
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode())
content_block_stop: Final = {"type": "content_block_stop", "index": index}
chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode())
return chunks

41
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response
_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool:
"""Return True when the provider should use the Responses API path.
def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
from litellm.main import responses_api_bridge_check
model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
resolved_model, custom_llm_provider
)
def _should_route_to_responses_api(
custom_llm_provider: str | None,
requested_model: str | None = None,
resolved_model: str | None = None,
) -> bool:
"""Return True when the request should use the Responses API path.
Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
opt out and route OpenAI/Azure requests through chat/completions instead.
"""
if litellm.use_chat_completions_url_for_anthropic_messages:
return False
return custom_llm_provider in _RESPONSES_API_PROVIDERS
if custom_llm_provider in _RESPONSES_API_PROVIDERS:
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
return False
return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
@ -533,7 +564,7 @@ def anthropic_messages_handler(
_shared_kwargs: Final = dict(
max_tokens=max_tokens,
messages=messages,
model=model,
model=original_model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
@ -551,7 +582,7 @@ def anthropic_messages_handler(
custom_llm_provider=custom_llm_provider,
**kwargs,
)
if _should_route_to_responses_api(custom_llm_provider):
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs)
# The in-gateway context_management polyfill runs inside

6
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -568,6 +568,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
custom_llm_provider=self._resolved_provider,
)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
optional_params=anthropic_messages_optional_request_params,
custom_llm_provider=self._resolved_provider,
)
self._translate_legacy_thinking_for_adaptive_model(
model=model,
optional_params=anthropic_messages_optional_request_params,

11
litellm/llms/anthropic/experimental_pass_through/messages/utils.py
View file

@ -44,6 +44,7 @@ class AnthropicMessagesRequestUtils:
filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None}
if model is not None:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
AnthropicConfig._maybe_drop_speed_param(
model=model,
@ -51,6 +52,16 @@ class AnthropicMessagesRequestUtils:
drop_params=drop_params,
custom_llm_provider=custom_llm_provider,
)
for param in ("temperature", "top_p", "top_k"):
if param in filtered_params:
AnthropicModelInfo._apply_sampling_param( # pyright: ignore[reportPrivateUsage] # same gating the /chat/completions path applies; forking it would drift
optional_params=filtered_params,
model=model,
param=param,
value=filtered_params.pop(param),
drop_params=drop_params,
output_key=param,
)
return cast(AnthropicMessagesRequestOptionalParams, filtered_params)

9
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
View file

@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
)
from litellm.types.llms.openai import ResponsesAPIResponse
from ..utils import local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
result: Final = await litellm.aresponses(**responses_kwargs)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
result: Final = litellm.responses(**responses_kwargs)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):

5
litellm/llms/anthropic/experimental_pass_through/utils.py
View file

@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
def local_model_name(model: str, custom_llm_provider: object) -> str:
"""The id the provider itself knows, for reporting back to the caller in ``message_start``."""
return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model
def is_reasoning_auto_summary_enabled() -> bool:
"""Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"

23
litellm/llms/azure/azure.py
View file

@ -17,7 +17,7 @@ from openai import (
import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -1352,6 +1352,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
organization: str | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
azure_ad_token: str | None = None,
azure_ad_token_provider: Callable | None = None,
aspeech: bool | None = None,
@ -1373,6 +1374,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
azure_ad_token_provider=azure_ad_token_provider,
max_retries=max_retries,
timeout=timeout,
logging_obj=logging_obj,
client=client,
litellm_params=litellm_params,
)
@ -1387,6 +1389,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
litellm_params=litellm_params,
)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(azure_client.base_url),
},
)
response: Final = azure_client.audio.speech.create(
model=model,
voice=voice,
@ -1408,6 +1419,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
azure_ad_token_provider: Callable | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
client=None,
litellm_params: dict | None = None,
) -> HttpxBinaryResponseContent:
@ -1421,6 +1433,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
litellm_params=litellm_params,
)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(azure_client.base_url),
},
)
azure_response: Final = await azure_client.audio.speech.create(
model=model,
voice=voice,

9
litellm/llms/base_llm/files/transformation.py
View file

@ -11,9 +11,9 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileContentRequest,
FileListPage,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LlmProviders, ModelResponse
@ -240,10 +240,13 @@ class BaseFileEndpoints(ABC):
@abstractmethod
async def afile_list(
self,
purpose: OpenAIFilesPurpose | None,
purpose: str | None,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
limit: int | None = None,
after: str | None = None,
**data: dict,
) -> list[OpenAIFileObject]:
) -> FileListPage:
pass
@abstractmethod

7
litellm/llms/bedrock/chat/converse_transformation.py
View file

@ -39,6 +39,7 @@ from litellm.llms.anthropic.chat.transformation import (
REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT,
AnthropicConfig,
)
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.request_metadata import (
bedrock_request_metadata_headers,
@ -1571,6 +1572,12 @@ class AmazonConverseConfig(BaseConfig):
"has no thinking_blocks. The model won't use extended thinking for this turn."
)
AnthropicModelInfo.maybe_drop_disabled_thinking(
model=model,
optional_params=optional_params,
custom_llm_provider="bedrock",
)
# Prepare and separate parameters
(
inference_params,

144
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -19,6 +19,10 @@ import litellm.types.utils
from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
@ -89,6 +93,7 @@ from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadCon
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -2056,7 +2061,7 @@ class BaseLLMHTTPHandler:
# Prepare headers
kwargs = kwargs or {}
provider_specific_header: Final = cast(
litellm.types.utils.ProviderSpecificHeader | None,
litellm.types.utils.ProviderSpecificHeader | Sequence[litellm.types.utils.ProviderSpecificHeader] | None,
kwargs.get("provider_specific_header", None),
)
provider_specific_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
@ -5077,9 +5082,12 @@ class BaseLLMHTTPHandler:
@staticmethod
def _get_agentic_loop_settings(kwargs: dict) -> tuple[int, int, list[str]]:
depth: Final = int(kwargs.get("_agentic_loop_depth", 0) or 0)
max_loops: Final = int(kwargs.get("max_agentic_loops", 3) or 3)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
fingerprints: Final = list(kwargs.get("_agentic_loop_fingerprints", []) or [])
return depth, max(max_loops, 1), fingerprints
return depth, max_loops, fingerprints
@staticmethod
def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool:
@ -5122,7 +5130,8 @@ class BaseLLMHTTPHandler:
"""
Evaluate agentic-loop safety guards (fingerprint cycle / max depth).
Raises ValueError on abort. Returns the current fingerprint on success.
Raises AgenticLoopSafetyError on abort. Returns the current fingerprint
on success.
These checks must not be swallowed by the per-callback ``except Exception``
block that wraps callback dispatch — they are bounded-loop / cycle-break
@ -5130,9 +5139,9 @@ class BaseLLMHTTPHandler:
"""
fingerprint: Final = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls)
if fingerprint in fingerprints:
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@staticmethod
@ -5142,6 +5151,97 @@ class BaseLLMHTTPHandler:
except Exception:
return str(tools)
@staticmethod
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
"""
Collect the ids and names of the tool calls a safety rail just refused.
Callbacks hand back either a bare list of tool calls or a dict wrapping
that list under ``tool_calls``, and both the anthropic and responses
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
"""
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
if not isinstance(calls, list):
return frozenset(), frozenset()
dict_calls: Final = (call for call in calls if isinstance(call, dict))
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
ids: Final = frozenset(
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
)
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
return ids, names
@staticmethod
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
"""
Whether this response block belongs to a tool call the rail refused.
An id settles it on its own, so a block carrying one is matched on the id
alone and a client's own tool call survives even where it happens to
share a name with a refused one. The name is only consulted for tool call
shapes that arrive without an id.
"""
if not isinstance(block, dict) or block.get("type") != "tool_use":
return False
block_id: Final = block.get("id")
if isinstance(block_id, str) and refused_ids:
return block_id in refused_ids
return block.get("name") in refused_names
@staticmethod
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
"""
Whether a refused rerun can still be answered with a finalized turn.
Only the anthropic messages surface can. The responses surface carries a
pydantic model the finalizer does not rewrite, so it keeps raising, which
is what every surface did before this path learned to end the turn.
The messages and responses call sites pass ``stream=False``, because
interception converts an intercepted stream to non-streaming before the
loop runs and rebuilds the SSE stream from the finalized turn
afterwards. ``AgenticStreamingIterator`` passes ``stream=True``, and
that path keeps raising: its events are already on the wire, so a
finalized turn would reach the client as a second message rather than
as a replacement.
"""
return not stream and api_surface == "anthropic_messages"
@staticmethod
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
"""
Turn the response into a terminal turn after a safety rail refused the rerun.
The refused tool calls target tools LiteLLM injected on the client's
behalf, so a client that never declared them cannot send back a matching
``tool_result``. Their blocks are dropped and a ``tool_use`` stop reason
is closed out as ``end_turn``, which is what a provider-native web search
turn returns once it stops calling tools.
A ``tool_use`` block the client itself declared is left alone, and while
one is still in the response the stop reason stays ``tool_use`` so the
client knows to answer it.
"""
if not isinstance(response, dict):
return response
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
finalized: Final = dict(response)
content: Final = finalized.get("content")
if isinstance(content, list):
kept_blocks: Final = [
block
for block in content
if not BaseLLMHTTPHandler._is_refused_tool_use_block(block, refused_ids, refused_names)
]
finalized["content"] = kept_blocks
client_tool_use_remains: Final = any(
isinstance(block, dict) and block.get("type") == "tool_use" for block in kept_blocks
)
if not client_tool_use_remains and finalized.get("stop_reason") == "tool_use":
finalized["stop_reason"] = "end_turn"
return finalized
async def _execute_anthropic_agentic_plan(
self,
plan: AgenticLoopPlan,
@ -5507,14 +5607,30 @@ class BaseLLMHTTPHandler:
continue
# Safety guards must run OUTSIDE the callback try/except — they are
# bounded-loop / cycle-break rails that must propagate to the caller.
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
# bounded-loop / cycle-break rails, not callback bugs.
try:
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
_call_id,
model,
str(e),
)
return self._maybe_wrap_in_fake_stream(
self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
logging_obj,
api_surface,
)
try:
kwargs_with_provider = hook_kwargs.copy()

51
litellm/llms/openai/openai.py
View file

@ -22,7 +22,7 @@ from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
@ -1365,9 +1365,21 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
client=client,
)
if headers:
data["extra_headers"] = headers
response = await openai_aclient.images.generate(**data, timeout=timeout)
logging_obj.pre_call(
input=prompt,
api_key=openai_aclient.api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"headers": {"Authorization": f"Bearer {openai_aclient.api_key}"}, # mutable-ok: logged header map
"api_base": str(openai_aclient.base_url),
"acompletion": True,
"complete_input_dict": data,
},
)
request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict
{**data, "extra_headers": headers} if headers else data
)
response = await openai_aclient.images.generate(**request_data, timeout=timeout)
stringified_response: Final = response.model_dump()
## LOGGING
logging_obj.post_call(
@ -1450,9 +1462,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
)
## COMPLETION CALL
if headers:
data["extra_headers"] = headers
_response: Final = openai_client.images.generate(**data, timeout=timeout)
request_data: Final = ( # mutable-ok: the OpenAI SDK takes the request body as a dict
{**data, "extra_headers": headers} if headers else data
)
_response: Final = openai_client.images.generate(**request_data, timeout=timeout)
response: Final = _response.model_dump()
## LOGGING
@ -1501,6 +1514,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
project: str | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
aspeech: bool | None = None,
client=None,
shared_session: Optional["ClientSession"] = None,
@ -1517,6 +1531,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
project=project,
max_retries=max_retries,
timeout=timeout,
logging_obj=logging_obj,
client=client,
shared_session=shared_session,
)
@ -1531,7 +1546,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
shared_session=shared_session,
)
response: Final = cast(OpenAI, openai_client).audio.speech.create(
sync_client: Final = cast(OpenAI, openai_client)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(sync_client.base_url),
},
)
response: Final = sync_client.audio.speech.create(
model=model,
voice=voice,
input=input,
@ -1551,6 +1576,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
project: str | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
client=None,
shared_session: Optional["ClientSession"] = None,
) -> HttpxBinaryResponseContent:
@ -1567,6 +1593,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
),
)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(openai_client.base_url),
},
)
response: Final = await openai_client.audio.speech.create(
model=model,
voice=voice,

12
litellm/llms/openai_like/providers.json
View file

@ -188,5 +188,17 @@
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
"api_key_env": "SCX_API_KEY",
"api_base_env": "SCX_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"constraints": {
"temperature_max": 1.99
},
"supported_endpoints": ["/v1/chat/completions"]
}
}

18
litellm/llms/vertex_ai/common_utils.py
View file

@ -1164,21 +1164,31 @@ class VertexAITokenCounter(BaseTokenCounter):
original_response=result,
)
else:
# Use standard Vertex AI (Gemini) token counter
from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
from litellm.llms.vertex_ai.gemini.transformation import (
_gemini_convert_messages_with_history, # pyright: ignore[reportPrivateUsage] # shared helper already used by gemini/chat, context_caching, and vertex_and_google_ai_studio_gemini
)
resolved_contents: Final = (
contents
if contents is not None
else _gemini_convert_messages_with_history(
messages=messages or [] # mutable-ok: fallback for None messages; helper signature requires list
)
)
count_tokens_params: Final = {
"model": model_to_use,
"contents": contents,
"contents": resolved_contents,
}
count_tokens_params_request.update(count_tokens_params)
result = await VertexAITokenCounter().acount_tokens(
**count_tokens_params_request,
)
if result is not None:
if result is not None and "totalTokens" in result:
return TokenCountResponse(
total_tokens=result.get("totalTokens", 0),
total_tokens=result["totalTokens"],
request_model=request_model,
model_used=model_to_use,
tokenizer_type=result.get("tokenizer_used", ""),

19
litellm/main.py
View file

@ -5091,14 +5091,16 @@ def completion(
model_info: Final = kwargs.get("model_info", None)
proxy_server_request: Final = kwargs.get("proxy_server_request", None)
fallbacks = kwargs.get("fallbacks", None)
provider_specific_header: Final = cast(ProviderSpecificHeader | None, kwargs.get("provider_specific_header", None))
provider_specific_header: Final = cast(
ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
kwargs.get("provider_specific_header", None),
)
headers = kwargs.get("headers", None) or extra_headers
ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None)
user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None)
assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None)
if headers is None:
headers = {}
headers = {} if headers is None else dict(headers)
if extra_headers is not None:
headers.update(extra_headers)
# Inject proxy auth headers if configured
@ -7535,6 +7537,15 @@ async def amoderation(
},
custom_llm_provider=custom_llm_provider,
)
moderation_request: Final = {"input": input, "model": model} # mutable-ok: logged as the raw request body
litellm_logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": moderation_request,
"api_base": str(_openai_client.base_url),
},
)
if model is not None:
response = await _openai_client.moderations.create(input=input, model=model)
@ -8040,6 +8051,7 @@ def speech(
project=project,
max_retries=max_retries,
timeout=timeout,
logging_obj=logging_obj,
client=client, # pass AsyncOpenAI, OpenAI client
aspeech=aspeech,
shared_session=shared_session,
@ -8118,6 +8130,7 @@ def speech(
organization=organization,
max_retries=max_retries,
timeout=timeout,
logging_obj=logging_obj,
client=client, # pass AsyncOpenAI, OpenAI client
aspeech=aspeech,
litellm_params=litellm_params_dict,

600
litellm/model_prices_and_context_window_backup.json
View file

@ -1232,6 +1232,7 @@
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"thinking_always_on": true,
"supports_function_calling": true,
"supports_vision": true,
"supports_prompt_caching": false,
@ -1404,6 +1405,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -1440,6 +1442,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -1476,6 +1479,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -1512,6 +1516,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -3021,6 +3026,7 @@
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -4875,6 +4881,38 @@
"supports_tool_choice": true,
"supports_vision": false
},
"azure/gpt-audio-mini": {
"deprecation_date": "2027-04-06",
"input_cost_per_audio_token": 1e-05,
"input_cost_per_token": 6e-07,
"litellm_provider": "azure",
"max_input_tokens": 128000,
"max_output_tokens": 16384,
"max_tokens": 16384,
"mode": "chat",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"supported_endpoints": [
"/v1/chat/completions"
],
"supported_modalities": [
"text",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_function_calling": true,
"supports_native_streaming": true,
"supports_parallel_function_calling": true,
"supports_prompt_caching": false,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": false
},
"azure/gpt-audio-mini-2025-10-06": {
"deprecation_date": "2027-04-06",
"input_cost_per_audio_token": 1e-05,
@ -5088,6 +5126,38 @@
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/gpt-realtime-mini": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
"input_cost_per_audio_token": 1e-05,
"input_cost_per_image": 8e-07,
"input_cost_per_token": 6e-07,
"litellm_provider": "azure",
"max_input_tokens": 32000,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "realtime",
"output_cost_per_audio_token": 2e-05,
"output_cost_per_token": 2.4e-06,
"supported_endpoints": [
"/v1/realtime"
],
"supported_modalities": [
"text",
"image",
"audio"
],
"supported_output_modalities": [
"text",
"audio"
],
"supports_audio_input": true,
"supports_audio_output": true,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"azure/gpt-realtime-mini-2025-10-06": {
"cache_creation_input_audio_token_cost": 3e-07,
"cache_read_input_token_cost": 6e-08,
@ -12780,6 +12850,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -19491,106 +19562,6 @@
},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_vision": true
},
"gemini/gemini-3.1-flash-lite-image": {
"rpm": 1000,
"tpm": 4000000,
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": true,
"supports_prompt_caching": false,
"supports_response_schema": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_vision": true
},
"vertex_ai/gemini-3.1-flash-lite-image": {
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_prompt_caching": true,
"supports_response_schema": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_vision": true
},
"gemini-3.1-flash-image": {
"deprecation_date": "2027-05-28",
"input_cost_per_image": 0.00056,
@ -19668,6 +19639,44 @@
},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true
},
"gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 5e-07,
@ -21498,6 +21507,42 @@
},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-image": {
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "gemini",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"rpm": 1000,
"source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": true,
"supports_prompt_caching": false,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_vision": true,
"tpm": 4000000
},
"gemini/deep-research-pro-preview-12-2025": {
"input_cost_per_image": 0.0011,
"input_cost_per_token": 2e-06,
@ -26034,33 +26079,33 @@
"supports_minimal_reasoning_effort": true
},
"gpt-5.6": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
"cache_creation_input_token_cost_flex": 3.125e-06,
"cache_creation_input_token_cost_priority": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_above_272k_tokens_flex": 5e-06,
"input_cost_per_token_batches": 2.5e-06,
"input_cost_per_token_flex": 2.5e-06,
"input_cost_per_token_priority": 1e-05,
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_flex": 2.5e-06,
"cache_creation_input_token_cost_priority": 1e-05,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost_above_272k_tokens": 8e-07,
"cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
"cache_read_input_token_cost_flex": 2e-07,
"cache_read_input_token_cost_priority": 8e-07,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
"input_cost_per_token_batches": 2e-06,
"input_cost_per_token_flex": 2e-06,
"input_cost_per_token_priority": 8e-06,
"litellm_provider": "openai",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
"output_cost_per_token_batches": 1.5e-05,
"output_cost_per_token_flex": 1.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token": 2e-05,
"output_cost_per_token_above_272k_tokens": 3e-05,
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
"output_cost_per_token_batches": 1e-05,
"output_cost_per_token_flex": 1e-05,
"output_cost_per_token_priority": 4e-05,
"regional_processing_uplift_multiplier_eu": 1.1,
"regional_processing_uplift_multiplier_us": 1.1,
"search_context_cost_per_query": {
@ -26097,33 +26142,33 @@
"supports_xhigh_reasoning_effort": true
},
"gpt-5.6-sol": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
"cache_creation_input_token_cost_flex": 3.125e-06,
"cache_creation_input_token_cost_priority": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
"cache_read_input_token_cost_flex": 2.5e-07,
"cache_read_input_token_cost_priority": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"input_cost_per_token_above_272k_tokens_flex": 5e-06,
"input_cost_per_token_batches": 2.5e-06,
"input_cost_per_token_flex": 2.5e-06,
"input_cost_per_token_priority": 1e-05,
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
"cache_creation_input_token_cost_flex": 2.5e-06,
"cache_creation_input_token_cost_priority": 1e-05,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost_above_272k_tokens": 8e-07,
"cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
"cache_read_input_token_cost_flex": 2e-07,
"cache_read_input_token_cost_priority": 8e-07,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"input_cost_per_token_above_272k_tokens_flex": 4e-06,
"input_cost_per_token_batches": 2e-06,
"input_cost_per_token_flex": 2e-06,
"input_cost_per_token_priority": 8e-06,
"litellm_provider": "openai",
"max_input_tokens": 922000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
"output_cost_per_token_batches": 1.5e-05,
"output_cost_per_token_flex": 1.5e-05,
"output_cost_per_token_priority": 6e-05,
"output_cost_per_token": 2e-05,
"output_cost_per_token_above_272k_tokens": 3e-05,
"output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
"output_cost_per_token_batches": 1e-05,
"output_cost_per_token_flex": 1e-05,
"output_cost_per_token_priority": 4e-05,
"regional_processing_uplift_multiplier_eu": 1.1,
"regional_processing_uplift_multiplier_us": 1.1,
"search_context_cost_per_query": {
@ -26365,19 +26410,19 @@
"supports_parallel_function_calling": true
},
"daybreak-blue-latest": {
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost": 5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1e-05,
"cache_read_input_token_cost": 4e-07,
"cache_read_input_token_cost_above_272k_tokens": 8e-07,
"input_cost_per_token": 4e-06,
"input_cost_per_token_above_272k_tokens": 8e-06,
"litellm_provider": "openai",
"max_input_tokens": 1050000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"output_cost_per_token": 2e-05,
"output_cost_per_token_above_272k_tokens": 3e-05,
"supported_endpoints": [
"/v1/chat/completions",
"/v1/responses"
@ -31140,6 +31185,23 @@
"supports_video_input": true,
"supports_vision": true
},
"moonshot/kimi-k3": {
"cache_read_input_token_cost": 3e-07,
"input_cost_per_token": 3e-06,
"litellm_provider": "moonshot",
"max_input_tokens": 1048576,
"max_output_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 1.5e-05,
"source": "https://platform.kimi.ai/docs/pricing/chat-k3",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_video_input": true,
"supports_vision": true
},
"moonshot/kimi-latest": {
"cache_read_input_token_cost": 1.5e-07,
"deprecation_date": "2026-01-28",
@ -36724,6 +36786,40 @@
"supports_vision": true,
"source": "https://cloud.sambanova.ai/plans/pricing"
},
"scx-ai/GLM-5.2": {
"cache_read_input_token_cost": 2.2e-07,
"input_cost_per_token": 6.1e-07,
"litellm_provider": "scx-ai",
"max_input_tokens": 1048576,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1.98e-06,
"source": "https://scx.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": false
},
"scx-ai/Qwen3.8-Max": {
"cache_read_input_token_cost": 2.1e-07,
"input_cost_per_token": 1.65e-06,
"litellm_provider": "scx-ai",
"max_input_tokens": 1000000,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 4.99e-06,
"source": "https://scx.ai/pricing",
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true
},
"snowflake/claude-3-5-sonnet": {
"litellm_provider": "snowflake",
"max_input_tokens": 200000,
@ -40365,6 +40461,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -40398,6 +40495,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -41006,6 +41104,44 @@
"supports_reasoning": false,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models"
},
"vertex_ai/gemini-3.1-flash-lite-image": {
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_image": 0.00028,
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-language-models",
"max_input_tokens": 65536,
"max_output_tokens": 4096,
"max_tokens": 4096,
"mode": "image_generation",
"output_cost_per_image": 0.0336,
"output_cost_per_image_token": 3e-05,
"output_cost_per_token": 1.5e-06,
"output_cost_per_token_batches": 7.5e-07,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
"/v1/completions",
"/v1/batch"
],
"supported_modalities": [
"text",
"image",
"video"
],
"supported_output_modalities": [
"text",
"image"
],
"supports_function_calling": false,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": false,
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true
},
"vertex_ai/gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
"input_cost_per_audio_token": 5e-07,
@ -48545,6 +48681,156 @@
"supports_tool_choice": true,
"supports_vision": true
},
"us.openai.gpt-5.6-sol": {
"input_cost_per_token": 5.5e-06,
"input_cost_per_token_above_272k_tokens": 1.1e-05,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
"cache_read_input_token_cost": 5.5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
"output_cost_per_token": 3.3e-05,
"output_cost_per_token_above_272k_tokens": 4.95e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"global.openai.gpt-5.6-sol": {
"input_cost_per_token": 5e-06,
"input_cost_per_token_above_272k_tokens": 1e-05,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
"cache_read_input_token_cost": 5e-07,
"cache_read_input_token_cost_above_272k_tokens": 1e-06,
"output_cost_per_token": 3e-05,
"output_cost_per_token_above_272k_tokens": 4.5e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"us.openai.gpt-5.6-terra": {
"input_cost_per_token": 2.2e-06,
"input_cost_per_token_above_272k_tokens": 4.4e-06,
"cache_creation_input_token_cost": 2.75e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
"cache_read_input_token_cost": 2.2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
"output_cost_per_token": 1.32e-05,
"output_cost_per_token_above_272k_tokens": 1.98e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"global.openai.gpt-5.6-terra": {
"input_cost_per_token": 2e-06,
"input_cost_per_token_above_272k_tokens": 4e-06,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_272k_tokens": 5e-06,
"cache_read_input_token_cost": 2e-07,
"cache_read_input_token_cost_above_272k_tokens": 4e-07,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_above_272k_tokens": 1.8e-05,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"us.openai.gpt-5.6-luna": {
"input_cost_per_token": 2.2e-07,
"input_cost_per_token_above_272k_tokens": 4.4e-07,
"cache_creation_input_token_cost": 2.75e-07,
"cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
"cache_read_input_token_cost": 2.2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
"output_cost_per_token": 1.32e-06,
"output_cost_per_token_above_272k_tokens": 1.98e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"global.openai.gpt-5.6-luna": {
"input_cost_per_token": 2e-07,
"input_cost_per_token_above_272k_tokens": 4e-07,
"cache_creation_input_token_cost": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens": 5e-07,
"cache_read_input_token_cost": 2e-08,
"cache_read_input_token_cost_above_272k_tokens": 4e-08,
"output_cost_per_token": 1.2e-06,
"output_cost_per_token_above_272k_tokens": 1.8e-06,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supported_modalities": [
"text",
"image"
],
"supported_output_modalities": [
"text"
],
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_vision": true
},
"bedrock_mantle/openai.gpt-5.5": {
"input_cost_per_token": 5.5e-06,
"cache_read_input_token_cost": 5.5e-07,
@ -49754,6 +50040,7 @@
},
"source": "https://docs.claude.com/en/docs/about-claude/models/overview",
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
@ -49789,6 +50076,7 @@
},
"source": "https://docs.claude.com/en/docs/about-claude/models/overview",
"supports_adaptive_thinking": true,
"thinking_always_on": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -49967,6 +50255,14 @@
"supports_adaptive_thinking": true
}
},
{
"name": "claude-always-on-thinking",
"pattern": "claude-(?:fable|mythos)-",
"description": "Any Claude Fable or Mythos id, under any provider namespace and any version. These families always think and reject thinking.type=disabled with a 400; the Anthropic transformations omit the param instead, so the model falls back to its default adaptive thinking.",
"model_info": {
"thinking_always_on": true
}
},
{
"name": "claude-mid-conversation-system",
"pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",

17
litellm/provider_endpoints_support_backup.json
View file

@ -2027,6 +2027,23 @@
"interactions": true
}
},
"scx-ai": {
"display_name": "SCX.ai (`scx-ai`)",
"url": "https://docs.litellm.ai/docs/providers/scx_ai",
"endpoints": {
"chat_completions": true,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": false,
"moderations": false,
"batches": false,
"rerank": false,
"a2a": false
}
},
"snowflake": {
"display_name": "Snowflake (`snowflake`)",
"url": "https://docs.litellm.ai/docs/providers/snowflake",

50
litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py
View file

@ -39,6 +39,7 @@ from litellm.proxy._types import (
SpecialMCPServerName,
SpecialMCPServerNames,
UserAPIKeyAuth,
user_api_key_has_admin_view,
)
from litellm.proxy.auth.ip_address_utils import IPAddressUtils
from litellm.proxy.auth.user_api_key_auth import (
@ -160,7 +161,7 @@ def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> b
"""True when this auth is a keyless subject admitted by the gateway session / bridge user
path, as opposed to a JWT or other keyless auth that merely lacks a ``team_id``.
Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``_reload_admitted_user``. It
Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``reload_admitted_user``. It
is deliberately NOT a ``metadata`` key, which is caller-controlled at key creation and so forgeable
on a personal key to gain the team grant union or dodge the egress scrub; this field cannot be."""
return user_api_key_auth is not None and user_api_key_auth.mcp_admitted_user_subject is True
@ -812,7 +813,7 @@ class MCPRequestHandler:
Identity-only sibling of :meth:`_admit_dcr_bridge_delegate`: the session token seals no
upstream credential (those are vaulted per user, resolved at egress), so authorization is
resolved fresh via :meth:`_reload_admitted_user` + the centralized policy gate rather than a
resolved fresh via :meth:`reload_admitted_user` + the centralized policy gate rather than a
mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard
pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired,
tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user."""
@ -835,7 +836,7 @@ class MCPRequestHandler:
match result:
case SessionBearerAdmitted():
try:
admitted: Final = await MCPRequestHandler._reload_admitted_user(result.principal.user_id)
admitted: Final = await MCPRequestHandler.reload_admitted_user(result.principal.user_id)
admitted.mcp_session_resource_server_id = result.principal.resource_server_id
await MCPRequestHandler._enforce_admitted_live_policy(
admitted=admitted, request=request, route=route
@ -893,12 +894,12 @@ class MCPRequestHandler:
case "key_hash":
return await MCPRequestHandler._reload_admitted_key(identity.subject)
case "user_id":
return await MCPRequestHandler._reload_admitted_user(identity.subject)
return await MCPRequestHandler.reload_admitted_user(identity.subject)
case _:
assert_never(identity.subject_type)
@staticmethod
async def _reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
async def reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
"""Reload the live user an interactively-minted envelope references and admit them as themselves.
The user's own object permission and ``org_id`` ride on the returned ``UserAPIKeyAuth``, and the
@ -1785,11 +1786,14 @@ class MCPRequestHandler:
global_mcp_server_manager,
)
# An OPEN channel (allow_all_keys, the user's own BYOM) makes the server REACHABLE through the
# user, though no grant source names it — without this the union returns [], listable but
# uninvokable. Reachability is ALL it confers, NOT a ceiling waiver: the user's own
# mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do on an allow_all server.
reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(auth)
# An OPEN channel (allow_all_keys, the user's own BYOM, an unscoped admin-view role) makes the
# server REACHABLE through the user, though no grant source names it — without this the union
# returns [], listable but uninvokable. Reachability is ALL it confers, NOT a ceiling waiver:
# the user's own mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do
# on an allow_all server or an admin key's do on any server.
reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(
auth
) or await MCPRequestHandler.admin_view_unscoped(auth)
allowed: Final[set[str]] = set()
for source, granted in await MCPRequestHandler.admitted_source_grants(auth):
@ -2723,6 +2727,32 @@ class MCPRequestHandler:
entitled_servers: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_user(user_api_key_auth)
return entitled_servers is None or len(entitled_servers) > 0
@staticmethod
async def admin_view_unscoped(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool:
"""Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
[] by DB default, so for that shape the row binds through the entitlement ceiling instead
(any non-empty entitlement, or an unresolved one, disqualifies), exactly as
``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
channel in ``_resolve_admitted_subject_tools`` both consult it, so the two axes cannot
disagree."""
if user_api_key_auth is None or not user_api_key_has_admin_view(user_api_key_auth):
return False
object_permission: Final = user_api_key_auth.object_permission
credential_scoped: Final = (
not _is_mcp_admitted_user_subject(user_api_key_auth)
and object_permission is not None
and object_permission.mcp_servers is not None
)
if credential_scoped:
return False
return not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
@staticmethod
async def _apply_user_tool_ceiling(
allowed_tools: Sequence[str] | None,

57
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -750,6 +750,55 @@ def _redirect_to_upstream_authorize(
return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params))))
def _bridge_access_denied_redirect(redirect_uri: str, state: str, mcp_server: MCPServer) -> RedirectResponse:
"""RFC 6749 section 4.1.2.1 denial for the interactive bridge authorize, delivered to the
already-validated client redirect_uri so a DCR client surfaces the failure at connect time."""
server_label: Final = mcp_server.alias or mcp_server.server_name or mcp_server.server_id
params: Final = {
"error": "access_denied",
"error_description": (
f"the signed-in user has no access to MCP server '{server_label}' on this gateway; "
"grant it through a team or user object permission, or mark the server allow_all_keys"
),
**({"state": state} if state else {}),
}
return RedirectResponse(_append_query_params(redirect_uri, params), status_code=302)
async def _bridge_authorize_access_denial(
litellm_user_id: str,
mcp_server: MCPServer,
redirect_uri: str,
state: str,
) -> RedirectResponse | None:
"""The denial redirect for a signed-in user who cannot reach the target server, or None to proceed.
Admits the user exactly as MCP egress will (the same ``reload_admitted_user`` constructor and the
same ``get_allowed_mcp_servers`` resolver), so an envelope is minted only when the resulting
session can actually list and call the server's tools. Without this gate the flow completes, the
client shows connected, and every tool request fail-closes to an empty list with nothing telling
the operator why. An availability fault (5xx, e.g. a DB outage's 503) propagates; an unknown or
deactivated user denies like a missing grant, fail closed.
"""
from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
try:
admitted: Final = await MCPRequestHandler.reload_admitted_user(litellm_user_id)
except HTTPException as exc:
if exc.status_code >= 500:
raise
return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
allowed_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(admitted)
if mcp_server.server_id in allowed_server_ids:
return None
return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
async def authorize_with_server(
request: Request,
mcp_server: MCPServer,
@ -819,6 +868,14 @@ async def authorize_with_server(
litellm_user_id = _user_id_from_session_cookie(request)
if litellm_user_id is None:
return _redirect_to_litellm_login(request)
denial: Final = await _bridge_authorize_access_denial(
litellm_user_id=litellm_user_id,
mcp_server=mcp_server,
redirect_uri=redirect_uri,
state=state,
)
if denial is not None:
return denial
encoded_state: Final = encode_state_with_base_url(
base_url=base_url,

41
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -2943,17 +2943,14 @@ class MCPServerManager:
2. If admin and no object_permission, return all servers
3. Otherwise, use standard permission checks
"""
from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
allow_all_server_ids: Final = self.get_allow_all_keys_server_ids()
# A keyless admitted subject is resolved per grant source, and channel decisions that are
# absolute for a scoped KEY credential are not absolute for it: its own opt-out silences its
# own source (handled per source in the resolver), never its teams' grants, and its admin
# role does not swallow the grant model — a session bearer is a third-party client
# credential, not the dashboard, so an admin signing in through the connect flow gets their
# grants like anyone else rather than handing the client the full registry ahead of every
# per-team org ceiling.
# own source (handled per source in the resolver), never its teams' grants. Its admin role
# rides the HUMAN, not the credential: an admin's session resolves the same registry their
# dashboard shows (connect-page parity), bounded like an admin key by explicit
# object_permission scope, the entitlement ceiling, and the session resource scope below.
is_admitted_subject: Final = _is_mcp_admitted_user_subject(user_api_key_auth)
# The key explicitly opted out of every MCP server. Return zero before
@ -2982,26 +2979,16 @@ class MCPServerManager:
)
try:
# If admin but NO explicit object permission, get all servers (never for an admitted
# subject — see is_admitted_subject above)
if (
user_api_key_auth
and not is_admitted_subject
and _user_has_admin_view(user_api_key_auth)
and not has_explicit_object_permission
# An entitlement attached to the HUMAN binds them whatever their role: it is the
# person's scope, not the credential's, so an admin role is not a waiver of it. An
# UNRESOLVED entitlement also skips the shortcut, so the resolver denies rather than
# handing over the whole registry on a transient fault.
and not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
):
verbose_logger.debug("Admin user without explicit object_permission - returning all servers")
return list(self.get_registry().keys())
# Get allowed servers from object permissions (respects object_permission even for admins)
allowed_mcp_servers: Final = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth)
verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers)
combined_servers: Final = set(allowed_mcp_servers)
# Admin view with no explicit object permission and no entitlement ceiling resolves the
# whole registry, for keys AND admitted session subjects alike (one predicate owns the
# question). Seeded into the union rather than returned early so the session resource
# scope below still bounds a per-server envelope held by an admin.
combined_servers: Final = (
set(self.get_registry().keys())
if await MCPRequestHandler.admin_view_unscoped(user_api_key_auth)
else set(await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth))
)
verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", combined_servers)
combined_servers.update(
await self.operator_open_server_ids(
user_api_key_auth,

18
litellm/proxy/_experimental/mcp_server/oauth_utils.py
View file

@ -180,6 +180,22 @@ def well_known_root_suffix() -> str:
return "" if root == "/" else root
def get_route_relative_request_path(scope: Scope) -> str:
"""The request path the MCP route shapes are written against: the raw ASGI path with the
deployment's ``root_path`` removed.
``scope["path"]`` and ``_original_path`` are both raw request-line paths, so on a sub-path
deployment they still carry the ``SERVER_ROOT_PATH`` prefix (``/litellm/{server}/mcp``) while
every route shape compared against them is root-relative. Mirrors the segment-boundary strip in
:func:`litellm.proxy.auth.auth_utils.get_request_route`, which the rest of the MCP auth path
already routes through, so ``/litellmfoo`` is not truncated under ``root_path=/litellm``."""
raw_path = str(scope.get("_original_path") or scope.get("path", "") or "")
root_path = str(scope.get("app_root_path") or scope.get("root_path") or "").rstrip("/")
if root_path and (raw_path == root_path or raw_path.startswith(f"{root_path}/")):
return raw_path[len(root_path) :]
return raw_path
def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str:
"""The per-server protected-resource metadata URL matching the spelling the request
arrived on, so a strict RFC 9728 client resolves the same route the proxy registered.
@ -188,7 +204,7 @@ def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str
the route decorators insert it (see :func:`well_known_root_suffix`)."""
request: Final = Request(scope)
base_url: Final = get_request_base_url(request)
_path: Final = scope.get("_original_path") or scope.get("path", "") or ""
_path: Final = get_route_relative_request_path(scope)
if _path.startswith(f"/{server_name}/mcp"):
return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp"

9
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -51,6 +51,8 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
from litellm.proxy._experimental.mcp_server.oauth_utils import (
_redact_mcp_resource_url,
get_passthrough_www_authenticate,
get_route_relative_request_path,
well_known_root_suffix,
)
from litellm.proxy._experimental.mcp_server.utils import (
LITELLM_MCP_SERVER_DESCRIPTION,
@ -3782,14 +3784,15 @@ if MCP_AVAILABLE:
request = StarletteRequest(scope)
base_url = get_request_base_url(request)
_path = scope.get("_original_path") or scope.get("path", "") or ""
_path = get_route_relative_request_path(scope)
# Pick the well-known AS-metadata form that matches the inbound route
# so strict RFC 9728 §3.2 clients can resolve it correctly.
as_metadata_root = f"{base_url}/.well-known/oauth-authorization-server{well_known_root_suffix()}"
if _path.startswith(f"/mcp/{server_name}"):
_as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}"
_as_url = f"{as_metadata_root}/mcp/{server_name}"
else:
_as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}"
_as_url = f"{as_metadata_root}/{server_name}"
authorization_uri = f'Bearer authorization_uri="{_as_url}"'
raise HTTPException(

2
litellm/proxy/_experimental/mcp_server/ui_session_utils.py
View file

@ -91,7 +91,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey
)
try:
admitted: Final = await MCPRequestHandler._reload_admitted_user(user_id)
admitted: Final = await MCPRequestHandler.reload_admitted_user(user_id)
except HTTPException as e:
verbose_logger.warning("MCP dashboard session: admitted-subject reload failed for %s: %s", user_id, e.detail)
return None

2
litellm/proxy/_experimental/out/404.html
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/404/index.html
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt
View file

@ -1,9 +1,9 @@
1:"$Sreact.fragment"
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js","/litellm-asset-prefix/_next/static/chunks/3l0glczkblv8_.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/2ca0bgyj3-r_j.js","/litellm-asset-prefix/_next/static/chunks/29nmr1sywlx25.js","/litellm-asset-prefix/_next/static/chunks/0gh1eppc9ekzh.js","/litellm-asset-prefix/_next/static/chunks/3hk5c4q5k-j7x.js","/litellm-asset-prefix/_next/static/chunks/2mhbxmykyh83f.js","/litellm-asset-prefix/_next/static/chunks/1xk5l9lxa0dv-.js","/litellm-asset-prefix/_next/static/chunks/26e7zpdybuhtq.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/0m-cn894wctv5.js","/litellm-asset-prefix/_next/static/chunks/3cw_k7_vr9pcu.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/2udc_95331vyv.js","/litellm-asset-prefix/_next/static/chunks/3mkd81u36rwju.js","/litellm-asset-prefix/_next/static/chunks/0kh9ov64og3-k.js","/litellm-asset-prefix/_next/static/chunks/3580ki1m5g-sx.js","/litellm-asset-prefix/_next/static/chunks/2xuwoxcnxuv39.js","/litellm-asset-prefix/_next/static/chunks/3bwziv83xzehe.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"OutletBoundary"]
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
7:"$Sreact.suspense"
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3l0glczkblv8_.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2ca0bgyj3-r_j.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/29nmr1sywlx25.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0gh1eppc9ekzh.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/3hk5c4q5k-j7x.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2mhbxmykyh83f.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1xk5l9lxa0dv-.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/26e7zpdybuhtq.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0m-cn894wctv5.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/3cw_k7_vr9pcu.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/2udc_95331vyv.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/3mkd81u36rwju.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0kh9ov64og3-k.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/3580ki1m5g-sx.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/2xuwoxcnxuv39.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/3bwziv83xzehe.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
4:{}
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
8:null

10
litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt
View file

@ -1,7 +1,7 @@
1:"$Sreact.fragment"
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

59
litellm/proxy/_experimental/out/__next._full.txt
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/__next._head.txt
View file

@ -1,6 +1,6 @@
1:"$Sreact.fragment"
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"MetadataBoundary"]
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
4:"$Sreact.suspense"
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

17
litellm/proxy/_experimental/out/__next._index.txt
View file

@ -1,10 +1,11 @@
1:"$Sreact.fragment"
2:I[12985,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"NuqsAdapter"]
3:I[867271,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
4:I[71195,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"AuthProvider"]
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

4
litellm/proxy/_experimental/out/__next._tree.txt
View file

@ -1,4 +1,4 @@
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","style"]
:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"TeJ852IBdcKgsOMzGKY73"}
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

0
litellm/proxy/_experimental/out/_next/static/TeJ852IBdcKgsOMzGKY73/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/TeJ852IBdcKgsOMzGKY73/_clientMiddlewareManifest.js → litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_clientMiddlewareManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/TeJ852IBdcKgsOMzGKY73/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/0cE25rDXvGu3tj4HWOGKy/_ssgManifest.js
View file

216
litellm/proxy/_experimental/out/_next/static/chunks/0013wgjn81q8k.js
View file

File diff suppressed because one or more lines are too long

Some files were not shown because too many files have changed in this diff Show more