scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

Merge upstream/litellm_internal_staging into bugfix/provider-budget-embedding-rerank

Browse source
This commit is contained in:
Apricooooot 2026-08-24 22:37:04 -07:00
commit 7e432ce217
2338 changed files with 43340 additions and 20901 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -651,126 +651,6 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -858,7 +738,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -985,7 +865,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -1030,7 +910,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1074,7 +954,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -1120,7 +1000,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -1211,7 +1091,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1255,7 +1135,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1274,40 +1154,6 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_enterprise_tests:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
batches_testing:
docker:
- *python312_image
@ -1333,7 +1179,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1377,7 +1223,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1422,7 +1268,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1501,7 +1347,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1546,7 +1392,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1599,7 +1445,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -3105,12 +2951,6 @@ workflows:
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:
@ -3167,8 +3007,6 @@ workflows:
filters: *main_branches
- search_testing:
filters: *main_branches
- litellm_mapped_enterprise_tests:
filters: *main_branches
- batches_testing:
filters: *main_branches
- litellm_utils_testing:
@ -3191,7 +3029,6 @@ workflows:
- guardrails_testing
- ocr_testing
- search_testing
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

1
.github/CODEOWNERS vendored
View file

@ -4,3 +4,4 @@
/model_prices_and_context_window.json @mateo-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri
/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
/.github/CODEOWNERS @yuneng-berri

31
.github/actions/cache-cargo-build/action.yml vendored Normal file
View file

@ -0,0 +1,31 @@
name: "Cache the Rust build"
description: >-
Cache the Cargo registry and target directory the root package's build needs,
so only the first job on a given Cargo.lock compiles the bridge from scratch.
litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
in release mode before it can produce a wheel. `uv sync` therefore pays a full
build in every job that installs the workspace: measured at 2m40s per unit shard
on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
it, because the uv cache holds wheels uv downloads rather than wheels it builds,
and a path dependency whose source moves every commit could never hit that cache
anyway. Cargo rebuilds only what changed when its target directory survives, so a
warm job pays for the bridge crate alone.
The key namespace is separate from test-rust.yml's. Both cache the same directory,
but that workflow fills it with debug and clippy artifacts, which a release build
cannot reuse, and a shared key would let whichever ran first deny the other a save.
runs:
using: composite
steps:
- name: Restore the Cargo registry and target directory
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-release-

46
.github/ci-coverage-allowlist.yml vendored
View file

@ -5,24 +5,21 @@ description: >-
test_paths:
- reason: >-
The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
2026-08-20 by collecting the directory under each job's own selector: 118 tests across
these eight files are selected by none of them. Listed so the gap is a decision rather
than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
when tests/local_testing is ported off CircleCI, where the keyless part of this suite
belongs in a real job
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
split that porting tests/local_testing off CircleCI will force, not a job that is red by
construction
paths:
- tests/local_testing/test_cache_preset_key.py
- tests/local_testing/test_caching.py
- tests/local_testing/test_caching_handler.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- tests/local_testing/test_prompt_caching.py
- tests/local_testing/test_responses_stream_cache_keys.py
- tests/local_testing/test_unit_test_caching.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds
@ -48,16 +45,6 @@ test_paths:
choice it informed is settled
paths:
- tests/code_coverage_tests/test_aio_http_image_conversion.py
- reason: >-
The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
their bodies into the live file of the same name. This one cannot follow either route yet:
its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
counterpart while 25 assertions fail against today's code, so what survives that rewrite
is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
behaviours still hold
paths:
- tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
- reason: >-
No job invokes this suite and its files mix pure transformation tests with ones driving live
vendor vector stores, so assigning them needs a per-file decision
@ -102,17 +89,6 @@ test_paths:
- tests/integration/sandbox/test_e2b_sandbox.py
- tests/integration/test_oci_integration.py
- tests/integration/test_oci_proxy_integration.py
- reason: >-
A unit test for the proxy-extras package that no job invokes, while the package's other
tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
this file requires. It also matches those keywords inside SQL comments, so two further
migrations are reported that are in fact fine. Wiring it up means deciding what to do about
the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
changed one breaks migrate deploy for existing installs
paths:
- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
dockerfiles:
- reason: >-

16
.github/scripts/assert_ci_coverage.py vendored
View file

@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
)
def _workflow_named_tokens() -> frozenset[str]:
"""Test tokens a GitHub Actions job names directly.
A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
workflow names it, so the slice check has to credit those the same way the census does.
"""
return _invoked_test_tokens(
scalar
for path in _config_files()
if path != CIRCLECI_CONFIG
for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
)
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
slices: Final = _slices()
named_by_workflow: Final = _workflow_named_tokens()
globbed: Final = tuple(
path
for path in _test_files()
@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
)
for path in globbed
if not allowlist.covers_test(path)
and not any(_token_covers(token, path) for token in named_by_workflow)
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
)

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,7 +27,7 @@ on:
default: 20
job-timeout-minutes:
description: >-
Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
the per-step ceilings on the setup steps below, and 5 for the runner
overhead the job clock charges but no step owns (job init, step
transitions, post-job cleanup). That headroom is what makes the test
@ -36,7 +36,7 @@ on:
arithmetic, so the sum is passed in rather than computed.
required: false
type: number
default: 55
default: 60
max-failures:
description: "Stop after this many failures"
required: false
@ -103,6 +103,11 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
@ -144,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -156,7 +161,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

4
.github/workflows/check-ui-api-types.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

33
.github/workflows/image-scan.yml vendored
View file

@ -17,10 +17,14 @@ on:
- backend/Dockerfile
- backend/main.py
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
- ui/nginx.conf
- .github/workflows/image-scan.yml
schedule:
- cron: "41 6 * * *"
@ -183,6 +187,35 @@ jobs:
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
ui-image:
name: ui-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build UI image
run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
env:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
runs-on: ubuntu-latest

3
.github/workflows/mutation-test.yml vendored
View file

@ -53,6 +53,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml

3
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -43,6 +43,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries

6
.github/workflows/test-code-quality.yml vendored
View file

@ -56,6 +56,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
@ -128,6 +131,9 @@ jobs:
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py

15
.github/workflows/test-linting.yml vendored
View file

@ -67,6 +67,17 @@ jobs:
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |
@ -78,6 +89,10 @@ jobs:
run: |
uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

6
.github/workflows/test-mcp.yml vendored
View file

@ -47,6 +47,10 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
@ -56,4 +60,4 @@ jobs:
- name: Run MCP tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

145
.github/workflows/test-postgres.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi

3
.github/workflows/test-terraform-provider.yml vendored
View file

@ -88,6 +88,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

4
.github/workflows/test-unit-documentation.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

53
.github/workflows/test-unit.yml vendored
View file

@ -55,7 +55,7 @@ jobs:
workers: 2
reruns: 1
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: enterprise-routing
artifact-name: enterprise-routing
@ -67,7 +67,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: integrations
artifact-name: integrations
@ -75,7 +75,7 @@ jobs:
workers: 2
reruns: 3
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: Vertex AI
artifact-name: llm-vertex-ai
@ -83,7 +83,7 @@ jobs:
workers: 1
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: All Other Providers
artifact-name: llm-other-providers
@ -91,7 +91,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: misc
artifact-name: misc
@ -122,7 +122,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-auth
artifact-name: proxy-auth
@ -134,7 +134,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-endpoints
artifact-name: proxy-endpoints
@ -168,10 +168,10 @@ jobs:
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-server
artifact-name: proxy-server
@ -179,7 +179,7 @@ jobs:
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
job-timeout-minutes: 100
- shard: proxy-infra
artifact-name: proxy-infra
@ -195,10 +195,39 @@ jobs:
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-extras
artifact-name: proxy-extras
test-path: "tests/litellm-proxy-extras"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-package
artifact-name: enterprise-package
test-path: "tests/enterprise"
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: responses-caching-types
artifact-name: responses-caching-types
@ -209,7 +238,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}

3
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -47,6 +47,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

2
CLAUDE.md
View file

@ -79,6 +79,8 @@ Do not put names of customers or customer company names in code, PR descriptions
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
- Composition over inheritance

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

10
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -966,6 +966,16 @@ class CheckBatchCost:
)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._finalize_unbilled_terminal_job(job, response)
# Record polling run metrics (always, even if nothing was processed)

87
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -45,6 +45,8 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
FILE_LIST_CONTINUATION_CHUNK_SIZE,
MAX_FILE_LIST_LIMIT,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
ensure_batch_response_managed_file_ids,
@ -54,6 +56,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
map_raw_file_ids_to_unified,
normalize_mime_type_for_provider,
resolve_managed_output_file_model_name,
validate_file_list_limit,
validate_file_list_purpose,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
@ -63,9 +67,9 @@ from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccess
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileListPage,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
ResponsesAPIResponse,
)
from litellm.types.utils import (
@ -144,7 +148,14 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
async def find_many(
self,
where: Mapping[str, object],
take: int = ...,
order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
cursor: Mapping[str, str] = ...,
skip: int = ...,
) -> Sequence[_ManagedFileRow]: ...
async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
@ -1365,12 +1376,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_list(
self,
purpose: Optional[OpenAIFilesPurpose],
purpose: Optional[str],
litellm_parent_otel_span: Optional[Span],
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
) -> FileListPage:
"""List the managed files the caller owns, newest first.
Pagination is keyset based on ``unified_file_id`` so a key that owns
every file on the proxy still reads one bounded page at a time.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
``data`` non-empty while matches remain and its last id usable as the
next cursor. A first chunk that fills the page costs one query; once a
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, with no index
covering either the owner filter or the sort.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
owner_filter: Final = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return FileListPage(**build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
where={**owner_filter, "unified_file_id": after}
)
if cursor_row is None:
raise ProxyException(
message=f"Invalid 'after' cursor: no file found with id '{after}'.",
type="invalid_request_error",
param="after",
code=400,
openai_code="invalid_value",
)
page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
matches: Final[List[OpenAIFileObject]] = []
cursor_id = after
chunk_size = page_size + 1
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
)
if len(chunk) < chunk_size:
break
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
def _is_batch_polling_enabled(self) -> bool:
"""

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.58"
version = "0.1.59"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.58"
version = "0.1.59"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

8
helm/litellm/values.yaml
View file

@ -428,9 +428,11 @@ ui:
maxUnavailable: ""
podAnnotations: {}
# Same shape as the gateway blocks of the same name. The nginx runtime
# writes its pid, cache, and proxy temp files under the image's root
# filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
# emptyDir volumes mounted over those paths.
# writes its pid, cache, and proxy temp files under /tmp, so it boots as
# any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
# true` here needs an emptyDir volume mounted over /tmp. Images before
# the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
# run as a non-root uid at all.
podLabels: {}
podSecurityContext: {}
securityContext: {}

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.88"
version = "0.4.89"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.88"
version = "0.4.89"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

24
litellm/_logging.py
View file

@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str:
return _redact_string(value)
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.
uvicorn's colorized formatter re-renders `color_message` against
record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead
of using the already-formatted record.msg, so it has to be substituted
before args are cleared or it is later formatted with no args and prints
the raw "%s://%s:%d" placeholders instead of the URL.
"""
color_message: Final = record.__dict__.get("color_message")
if not isinstance(color_message, str) or not record.args:
return None
try:
return color_message % record.args
except TypeError:
return color_message
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter):
if not _ENABLE_SECRET_REDACTION:
return True
# Runs before args are cleared, and before the extra-field loop below
# that redacts the substituted result.
substituted_color_message: Final = _substituted_color_message(record)
if substituted_color_message is not None:
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
try:
record.msg = _redact_string(record.getMessage())
record.args = None

10
litellm/_redis.py
View file

@ -665,8 +665,16 @@ def get_redis_async_client(
cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
cluster_kwargs.setdefault("socket_keepalive", True)
# A single node's client-side timeout must reset only that node's connections,
# not tear down the whole cluster client for every concurrent caller.
from litellm.caching.redis_cluster_node_isolation import (
get_litellm_async_redis_cluster_class,
)
async_redis_cluster_class: Final = get_litellm_async_redis_cluster_class()
# Create async RedisCluster with IAM token as password if available
cluster_client: Final = async_redis.RedisCluster(
cluster_client: Final = async_redis_cluster_class(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)

38
litellm/a2a_protocol/card_resolver.py
View file

@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
{
"jsonrpc": "JSONRPC",
"http+json": "HTTP+JSON",
"grpc": "GRPC",
}
)
_LEGACY_PROTOCOL_VERSION: Final = "0.3"
def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
"""
Canonicalize the supported interfaces of spec-adjacent agent cards.
Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
case-sensitively against its uppercase TransportProtocol constants and fails
with "no compatible transports found." for spec-adjacent casings.
The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
proto parsing rejects. A mis-cased binding fingerprints such a server, so its
declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
its v0.3 compat transport, which speaks that dialect.
"""
normalized: Final = type(agent_card)()
normalized.CopyFrom(agent_card)
for interface in normalized.supported_interfaces:
canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
if canonical is None or canonical == interface.protocol_binding:
continue
interface.protocol_binding = canonical
interface.protocol_version = _LEGACY_PROTOCOL_VERSION
return normalized
def get_agent_card_url(agent_card: "AgentCard") -> str | None:
"""Return the agent endpoint URL from the resolved SDK card."""
url: Final = getattr(agent_card, "url", None)

13
litellm/a2a_protocol/main.py
View file

@ -73,6 +73,7 @@ except ImportError:
from litellm.a2a_protocol.card_resolver import (
LiteLLMA2ACardResolver,
get_agent_card_url,
normalize_agent_card_interfaces,
)
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
@ -782,13 +783,17 @@ async def create_a2a_client(
if extra_headers:
verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card: Final = normalize_agent_card_interfaces(
await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
)
a2a_client: Final = await create_client( # pyright: ignore[reportOptionalCall]
base_url,
agent_card,
client_config=ClientConfig( # pyright: ignore[reportOptionalCall]
httpx_client=httpx_client,
streaming=streaming,
),
resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
)
# Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
# the configured httpx client and this agent's headers without excavating
@ -799,9 +804,7 @@ async def create_a2a_client(
if extra_headers
else None
)
agent_card: Final = getattr(a2a_client, "_card", None)
if agent_card is not None:
a2a_client._litellm_agent_card = agent_card
a2a_client._litellm_agent_card = agent_card
verbose_logger.info("A2A client created for %s", base_url)

40
litellm/batches/batch_utils.py
View file

@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage(
)
def _provider_output_file_id(output_file_id: str) -> str:
"""
Resolve the file id the provider actually knows: unified ids yield their embedded
llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_original_file_id,
)
unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id)
if not unified_file_id:
return get_original_file_id(output_file_id)
try:
extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s",
output_file_id,
e,
)
return output_file_id
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted)
return extracted
async def _fetch_batch_output_file_content(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content(
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
if batch.output_file_id is None:
raise ValueError("Output file id is None cannot retrieve file content")
file_id = batch.output_file_id
is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
try:
file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id)
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e
)
file_id: Final = _provider_output_file_id(batch.output_file_id)
# Build kwargs for afile_content with credentials from litellm_params
file_content_kwargs: Final = {

173
litellm/caching/redis_cluster_node_isolation.py Normal file
View file

@ -0,0 +1,173 @@
"""Bounds the blast radius of a single node's transient connection error on the async
Redis Cluster client.
redis-py's ``RedisCluster._execute_command`` responds to a ``ConnectionError`` or
``TimeoutError`` on ANY one node by tearing down every node's connections and flipping
the client into "needs reinitialization", which forces every other concurrent caller
sharing this client through one reinit lock until the whole cluster topology is
re-walked. Under real proxy load, a client-side socket timeout on a single node is a
routine event (the event loop was too busy to read the response before ``socket_timeout``
elapsed) and does not mean the cluster's topology moved, so treating it as a full-cluster
event turns one slow node into a proxy-wide latency spike while Redis itself stays
healthy -- confirmed live: pausing one of three local cluster nodes made every concurrent
command against the other two, untouched nodes stall for the full pause duration too.
``get_litellm_async_redis_cluster_class`` returns a ``RedisCluster`` subclass that resets
only the node that actually failed (mirroring what a plain, non-cluster Redis client
already does when one of its pooled connections errors), leaving every other node's
connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
topology changed.
"""
import asyncio
from typing import TYPE_CHECKING, Final, Protocol
from litellm._logging import verbose_logger
if TYPE_CHECKING:
from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
class _ClusterNodeAttrs(Protocol):
"""The subset of ``redis.asyncio.cluster.ClusterNode`` this override reads. redis-py
ships no resolvable stub for these members under the repo's current types-redis pin,
so a plain attribute access resolves every downstream use to ``Unknown`` under strict
mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
own logic fully typed without a banned ``typing.cast``."""
async def execute_command(
self,
*args: object,
**kwargs: object, # kwargs-ok: mirrors redis-py's own ClusterNode.execute_command signature, a raw command dispatch with no fixed keyword contract
) -> object: ...
async def disconnect(self) -> None: ...
class _NodesManagerAttrs(Protocol):
_moved_exception: object
def get_node_from_slot(
self, slot: int, read_from_replicas: bool, load_balancing_strategy: object
) -> _ClusterNodeAttrs: ...
class _ClusterAttrs(Protocol):
RedisClusterRequestTTL: int
reinitialize_counter: int
reinitialize_steps: int
read_from_replicas: bool
load_balancing_strategy: object
nodes_manager: _NodesManagerAttrs
def get_node(self, node_name: str) -> _ClusterNodeAttrs: ...
async def _determine_slot(self, *args: object) -> int: ...
async def aclose(self) -> None: ...
#: redis-py versions this override's copied ``_execute_command`` body has been verified
#: against. A version outside this set may have changed the method's structure in a way
#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
#: construction logs a loud warning rather than silently trusting an unverified copy.
_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
"""Builds the ``RedisCluster`` subclass with the per-node isolation fix.
Imported lazily because this module is reachable from a base ``import litellm`` while
redis is not a base dependency. Cheap to call repeatedly: the underlying redis
submodules are cached in ``sys.modules`` after the first import.
"""
import redis
from redis.asyncio.cluster import (
RedisCluster as _BaseAsyncRedisCluster, # pyright: ignore[reportUnknownVariableType] # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
)
from redis.cluster import get_node_name
from redis.commands import READ_COMMANDS
from redis.exceptions import (
AskError,
BusyLoadingError,
ClusterDownError,
ClusterError,
MaxConnectionsError,
MovedError,
SlotNotCoveredError,
TryAgainError,
)
from redis.exceptions import ConnectionError as _RedisConnectionError
from redis.exceptions import TimeoutError as _RedisTimeoutError
if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
verbose_logger.warning(
"redis-py %s is not in the set this cluster-teardown-storm fix was verified "
"against (%s). The per-node-isolation override may not match the installed library's "
"real _execute_command behavior.",
redis.__version__,
sorted(_VERIFIED_REDIS_VERSIONS),
)
class LiteLLMAsyncRedisCluster(
_BaseAsyncRedisCluster # pyright: ignore[reportUntypedBaseClass] # same stale-stub gap as the import above; the base class itself is unresolvable, not this subclass's own code
):
async def _execute_command(
self,
target_node: _ClusterNodeAttrs,
*args: object,
**kwargs: object, # kwargs-ok: overrides redis-py's own **kwargs signature; the keyword contract is defined by the Redis command being dispatched, not by this method
) -> object:
cluster: _ClusterAttrs = self
node = target_node
asking = moved = False
redirect_addr: str | None = None
ttl = cluster.RedisClusterRequestTTL
while ttl > 0:
ttl -= 1
try:
if asking:
assert redirect_addr is not None
node = cluster.get_node(node_name=redirect_addr)
await node.execute_command("ASKING")
asking = False
elif moved:
slot = await cluster._determine_slot(*args) # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch, which makes this identical private call from the same subclass
node = cluster.nodes_manager.get_node_from_slot(
slot,
cluster.read_from_replicas and args[0] in READ_COMMANDS,
(cluster.load_balancing_strategy if args[0] in READ_COMMANDS else None),
)
moved = False
return await node.execute_command(*args, **kwargs)
except (BusyLoadingError, MaxConnectionsError):
raise
except (_RedisConnectionError, _RedisTimeoutError):
# Reset only the node that actually failed instead of the upstream
# default (`await self.aclose()`, a full-cluster teardown that forces
# every other concurrent caller through the shared reinit lock).
await node.disconnect()
raise
except (ClusterDownError, SlotNotCoveredError):
await cluster.aclose()
await asyncio.sleep(0.25)
raise
except MovedError as e:
cluster.reinitialize_counter += 1
if cluster.reinitialize_steps and cluster.reinitialize_counter % cluster.reinitialize_steps == 0:
await cluster.aclose()
cluster.reinitialize_counter = 0
else:
cluster.nodes_manager._moved_exception = e # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch; redis-py exposes no public setter for this
moved = True
except AskError as e:
redirect_addr = get_node_name(host=e.host, port=e.port)
asking = True
except TryAgainError:
if ttl < cluster.RedisClusterRequestTTL / 2:
await asyncio.sleep(0.05)
raise ClusterError("TTL exhausted.")
return LiteLLMAsyncRedisCluster

37
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -21,6 +21,9 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_item_from_thinking_blocks,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
@ -85,6 +88,22 @@ def _get_reasoning_items(
return []
def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload
"""Reasoning input items for an assistant message.
Stored reasoning items win because they carry an id the Responses API minted; thinking
blocks are the fallback for turns that arrived over another API surface.
"""
items: Final = _get_reasoning_items(msg)
stored: Final = [_reasoning_item_to_response_input(item) for item in items] # mutable-ok: API message payload
if stored:
return stored
raw_blocks: Final = msg.get("thinking_blocks") or ()
blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json
from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload
def _build_reasoning_item(
item_id: str,
encrypted_content: str | None,
@ -372,8 +391,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
if content:
input_items.append(
{ # mutable-ok: API message payload
"type": "message",
"role": "assistant",
"content": self._convert_content_to_responses_format(content, "assistant"),
}
)
for tool_call in tool_calls:
function = tool_call.get("function")
custom = tool_call.get("custom")
@ -400,15 +426,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raise ValueError(f"tool call not supported: {tool_call}")
elif content is not None:
if role == "assistant":
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
input_items.append(
{
{ # mutable-ok: API message payload
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
elif role == "assistant":
input_items.extend(_reasoning_input_items(msg))
return input_items, instructions

17
litellm/constants.py
View file

@ -783,6 +783,7 @@ openai_compatible_endpoints: Final[list] = [
"https://pinstripes.io/v1",
"https://api.meta.ai/v1",
"https://api.cognition.ai/v1",
"https://api.scx.ai/v1",
]
@ -851,6 +852,7 @@ openai_compatible_providers: Final[list] = [
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
"scx-ai",
]
openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions`
"together_ai",
@ -1540,6 +1542,8 @@ SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BA
SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3")))
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2"))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
@ -1559,6 +1563,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
# installations with large numbers of stale managed objects).
_batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
)
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
)
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
)
_background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

25
litellm/cost_calculator.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
TranscriptionUsageObjectTransformation,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import (
@ -150,6 +151,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
}
)
_SPEECH_CALL_TYPES: Final = frozenset(
{
CallTypes.speech.value,
@ -912,6 +914,8 @@ def _get_usage_object(
usage_obj,
)
)
elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
@ -1288,6 +1292,10 @@ def completion_cost(
)
if tr_usage is not None:
_usage = tr_usage.model_dump()
elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
_usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
_usage
).model_dump()
else:
_usage = _usage
@ -1372,23 +1380,36 @@ def completion_cost(
if custom_pricing and litellm_logging_obj is not None:
_litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
if _litellm_params is not None:
_metadata = _litellm_params.get("metadata", {}) or {}
_video_model_info = _metadata.get("model_info", None)
_video_model_info = next(
(
model_info
for _metadata_key in ("metadata", "litellm_metadata")
if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
is not None
),
None,
)
usage_obj = getattr(completion_response, "usage", None)
duration_seconds: float | None = None
video_resolution: str | None = None
provider_reported_cost: float | None = None
if completion_response is not None and usage_obj:
# Handle both dict and Pydantic Usage object
if isinstance(usage_obj, dict):
duration_seconds = usage_obj.get("duration_seconds", None)
_vr = usage_obj.get("video_resolution", None)
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
else:
duration_seconds = getattr(usage_obj, "duration_seconds", None)
_vr = getattr(usage_obj, "video_resolution", None)
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
if _vr is not None:
video_resolution = str(_vr).strip().lower()
if _video_model_info is None and provider_reported_cost is not None:
return float(provider_reported_cost)
if duration_seconds is not None:
# Calculate cost based on video duration using video-specific cost calculation
from litellm.llms.openai.cost_calculation import (

46
litellm/images/main.py
View file

@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -422,24 +423,32 @@ def image_generation(
aimg_generation=aimg_generation,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
api_base = AzureFoundryModelInfo.get_api_base(api_base)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
default_headers = {
caller_header_names = frozenset(name.lower() for name in headers)
caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
auth_headers = (
headers
if caller_set_auth
else get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params_dict,
api_key_header="api-key",
)
)
request_headers: Final = {
"Content-Type": "application/json",
**auth_headers,
**headers,
}
# Only add api-key header if api_key is not None
# Azure AD authentication will use Authorization header instead
if api_key is not None:
default_headers["api-key"] = api_key
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
@ -455,7 +464,7 @@ def image_generation(
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
headers=request_headers,
litellm_params=litellm_params_dict,
)
elif (
@ -846,6 +855,18 @@ def image_edit(
additional_drop_params=kwargs.get("additional_drop_params"),
)
if (
custom_llm_provider == "openai"
or custom_llm_provider == "azure"
or custom_llm_provider in litellm.openai_compatible_providers
):
image_edit_request_params.update(
flatten_form_field_values(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
)
)
# Pre Call logging
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
@ -995,6 +1016,9 @@ async def aimage_edit(
response_format=response_format,
size=size,
user=user,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,

14
litellm/integrations/callback_configs.json
View file

@ -294,12 +294,18 @@
"id": "newrelic",
"displayName": "New Relic",
"logo": "newrelic.png",
"supports_key_team_logging": false,
"supports_key_team_logging": true,
"dynamic_params": {
"NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
"newrelic_api_key": {
"type": "password",
"ui_name": "New Relic Ingest License Key",
"description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
"required": false
},
"newrelic_region": {
"type": "text",
"ui_name": "Record AI Content (default: true)",
"description": "Whether to record AI message content. Set to false to disable.",
"ui_name": "New Relic Region (us or eu)",
"description": "Data center region for this team's account. Defaults to us.",
"required": false
}
},

22
litellm/integrations/langsmith.py
View file

@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
return outputs
def _ensure_required_ids(self, data: dict, run_id: str | None):
resolved_id: Final = run_id or str(uuid.uuid4())
if "id" not in data or data["id"] is None:
run_id = str(uuid.uuid4())
data["id"] = run_id
data["id"] = resolved_id
if "trace_id" not in data or data["trace_id"] is None:
if run_id is not None and isinstance(run_id, str):
data["trace_id"] = run_id
# LangSmith rejects the whole ingest batch unless a root run's trace_id
# equals the run id embedded in the first segment of dotted_order
posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
"dotted_order" not in data or data["dotted_order"] is None
)
if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
data["trace_id"] = resolved_id
if "dotted_order" not in data or data["dotted_order"] is None:
if run_id is not None and isinstance(run_id, str):
data["dotted_order"] = self.make_dot_order(run_id=run_id)
data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
def _prepare_log_data(
self,
@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
metadata = _litellm_params.get("metadata", {}) or {}
fields: Final = self._extract_metadata_fields(metadata, credentials)
# the proxy header fan-out mirrors one value into both keys, and LangSmith
# rejects the whole ingest batch when run-body session_id is not an
# existing tracer-session uuid
if fields["session_id"] == fields["trace_id"]:
fields["session_id"] = None
verbose_logger.debug(
"Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
)

4
litellm/integrations/otel/__init__.py
View file

@ -49,6 +49,7 @@ from litellm.integrations.otel.model.semconv import (
Error,
GenAI,
GenAIOperation,
GenAIOutputType,
GenAIProvider,
JsonRpc,
LiteLLM,
@ -60,6 +61,7 @@ from litellm.integrations.otel.model.semconv import (
RpcSystem,
Server,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.spans import (
@ -84,6 +86,7 @@ __all__ = [
"Error",
"GenAI",
"GenAIOperation",
"GenAIOutputType",
"GenAIProvider",
"GuardrailSpanData",
"JsonRpc",
@ -116,6 +119,7 @@ __all__ = [
"is_otel_v2_enabled",
"promoted_baggage",
"resolve_operation",
"resolve_output_type",
"resolve_provider",
"span_role_for_service",
"validate_registry",

6
litellm/integrations/otel/emitter.py
View file

@ -156,6 +156,12 @@ class SpanEmitter:
links=list(links) if links else None,
)
def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
"""Register a span emitted outside :meth:`emit` (the boundary-opened
LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
for the same ``(dedup_key, role)`` deduplicates against it."""
self._seen(dedup_key, role)
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
"""Return True once a ``(dedup_key, role)`` pair has been emitted.

36
litellm/integrations/otel/logger.py
View file

@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
if carrier is None:
# A missing carrier does not always mean nothing happened: a team/key-scoped
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
return None
try:
return self._finish_carrier(carrier, call, end_time)
return self._finish_carrier(carrier, call, start_time, end_time)
finally:
# After the span has ended, so a release-triggered provider shutdown
# force-flushes it out rather than racing its enqueue.
@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
"""Remember an in-flight LLM call, evicting the oldest if over budget.
A call that opens but never closes (a stream that only fires stream
events) would linger otherwise; the evicted span is simply dropped
(never exported).
events) would linger otherwise. Eviction only drops the boundary carrier,
not the call: if that call later closes as a real completed call, it still
emits through the deferred branch in ``_close_llm_call`` (the same path a
team/key-scoped logger uses, since it never opens a carrier), deduplicated
by call id. Only a call that is evicted and never closes goes unexported.
"""
self._open_llm_calls[call_id] = carrier
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
def _finish_carrier(
self,
carrier: _LLMCallSpan,
carrier: "_LLMCallSpan | None",
call: LLMCallEvent,
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
payload: Final = call.payload
call_id: Final = call.call_id
if payload is None:
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Opened at the boundary but the payload never materialized — end
# it (named provisionally) so it isn't leaked as an open span.
# it (named provisionally) so it isn't leaked as an open span, and
# register the dedup marker so a later payload-carrying close for
# the same call id cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
carrier.span.end(end_time=to_ns(end_time))
return None
data: Final = LLMCallSpanData.from_standard_logging_payload(
@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
)
end_time_ns: Final = to_ns(end_time)
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set
# status, and end it. Its parent (the server span) was captured at
# creation from real ambient context.
# creation from real ambient context. Register the dedup marker so a
# second close for the same call id (success then failure on one
# logging object) cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
return carrier.span
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
SpanRole.LLM_CALL,
data,
parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
start_time_ns=carrier.start_time_ns,
start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
end_time_ns=end_time_ns,
tracer=route.tracer,
links=_request_trace_links(parent_ctx) if route.detached else None,

2
litellm/integrations/otel/mappers/genai.py
View file

@ -42,6 +42,7 @@ class GenAIMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
GenAI.OPERATION_NAME: lambda d: d.operation.value,
GenAI.PROVIDER_NAME: lambda d: d.provider or None,
GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
@ -65,6 +66,7 @@ class GenAIMapper:
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
LiteLLM.CALL_TYPE: lambda d: d.call_type,
# The provider/underlying model is only known once routing has picked a
# deployment, so it can't ride identity Baggage (seeded at auth, before
# routing) onto the boundary-born LLM span — stamp it directly here.

10
litellm/integrations/otel/model/config.py
View file

@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
WEAVE_OTEL = "weave_otel"
LEVO = "levo"
AGENTOPS = "agentops"
NEWRELIC = "newrelic"
class _OTelV2Flag(BaseSettings):
@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
"auto (Simple for console/in_memory, Batch otherwise)."
),
)
requires_headers: bool = Field(
default=False,
description=(
"Skip this exporter when no headers are resolved. For destinations "
"that reject unauthenticated exports (e.g. New Relic), a spec kept "
"only as the per-request credential-stamping target would otherwise "
"export keyless traffic and produce a 4xx for every span batch."
),
)
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -203,6 +203,11 @@ class LLMCallEvent:
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# True once the request handed off to a provider (``pre_call`` stamped
# ``api_call_start_time``). The affirmative signal that an LLM call was
# actually attempted — router pre-call rejections, SDK failures before the
# provider handoff, and standalone guardrail runs all lack it.
upstream_started: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -221,6 +226,7 @@ class LLMCallEvent:
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
auth_metadata=auth_metadata(payload, kwargs),
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
upstream_started=kwargs.get("api_call_start_time") is not None,
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
)

12
litellm/integrations/otel/model/payloads.py
View file

@ -15,8 +15,10 @@ from litellm.integrations.otel.model.metadata import (
)
from litellm.integrations.otel.model.semconv import (
GenAIOperation,
GenAIOutputType,
MCPMethod,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.utils import (
@ -310,6 +312,11 @@ class LLMCallSpanData:
choices_out: tuple[Mapping[str, object], ...] = ()
system_fingerprint: str | None = None
time_to_first_chunk_seconds: float | None = None
# The requested output modality, set only on the routes that pin one (image
# generation, speech, transcription, OCR), and the litellm route itself, which
# keeps routes the convention folds into one operation distinguishable.
output_type: GenAIOutputType | None = None
call_type: str | None = None
@classmethod
def from_standard_logging_payload(
@ -334,8 +341,9 @@ class LLMCallSpanData:
# otherwise the content-bearing mappers receive empty sequences and emit
# no prompt/response text.
finish_reasons: Final = _finish_reasons(choices_out)
call_type: Final = as_str(payload.get("call_type"))
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
operation=resolve_operation(call_type),
provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
request_model=context.request_model,
response_model=context.response_model,
@ -358,6 +366,8 @@ class LLMCallSpanData:
choices_out=choices_out if capture_content else (),
system_fingerprint=as_str(response.get("system_fingerprint")),
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
output_type=resolve_output_type(call_type),
call_type=call_type or None,
)

57
litellm/integrations/otel/model/semconv.py
View file

@ -3,7 +3,9 @@ Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anythin
without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
"""
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
@ -30,6 +32,21 @@ class GenAIOperation(str, Enum):
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_MODERATION = "litellm.moderation"
class GenAIOutputType(str, Enum):
"""Values for ``gen_ai.output.type``, the modality the client asked for.
It is what separates the inference routes that share ``generate_content``:
image generation requests ``image``, speech requests ``speech``, and
transcription and OCR both request ``text``.
"""
TEXT = "text"
JSON = "json"
IMAGE = "image"
SPEECH = "speech"
class GenAIProvider(str, Enum):
@ -258,6 +275,11 @@ class LiteLLM:
"""Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
CALL_ID: Final = "litellm.call_id"
# The litellm route that produced the call. Needed because the convention maps
# several routes onto one operation: transcription and OCR are both
# ``generate_content`` with a ``text`` output type, so this is the only thing
# that tells them apart.
CALL_TYPE: Final = "litellm.call_type"
COST_PREFIX: Final = "litellm.cost."
METADATA_PREFIX: Final = "litellm.metadata."
TEAM_ID: Final = "litellm.team.id"
@ -352,6 +374,16 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"image_generation": GenAIOperation.GENERATE_CONTENT,
"aimage_generation": GenAIOperation.GENERATE_CONTENT,
"moderation": GenAIOperation.LITELLM_MODERATION,
"amoderation": GenAIOperation.LITELLM_MODERATION,
"ocr": GenAIOperation.GENERATE_CONTENT,
"aocr": GenAIOperation.GENERATE_CONTENT,
"speech": GenAIOperation.GENERATE_CONTENT,
"aspeech": GenAIOperation.GENERATE_CONTENT,
"transcription": GenAIOperation.GENERATE_CONTENT,
"atranscription": GenAIOperation.GENERATE_CONTENT,
"call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
"vector_store_search": GenAIOperation.RETRIEVAL,
"avector_store_search": GenAIOperation.RETRIEVAL,
@ -385,6 +417,23 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
}
# litellm ``call_type`` -> ``gen_ai.output.type``. Only the call types whose route
# fixes the requested modality are listed; the attribute is conditionally required
# on a request that asks for an output format, so anything else is left unstamped.
_OUTPUT_TYPE_BY_CALL_TYPE: Final[Mapping[str, GenAIOutputType]] = MappingProxyType(
{
"image_generation": GenAIOutputType.IMAGE,
"aimage_generation": GenAIOutputType.IMAGE,
"speech": GenAIOutputType.SPEECH,
"aspeech": GenAIOutputType.SPEECH,
"transcription": GenAIOutputType.TEXT,
"atranscription": GenAIOutputType.TEXT,
"ocr": GenAIOutputType.TEXT,
"aocr": GenAIOutputType.TEXT,
}
)
def resolve_provider(custom_llm_provider: str | None) -> str:
"""Map a litellm provider string to a ``gen_ai.provider.name`` value.
@ -416,3 +465,11 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
GenAIOperation.CHAT.value,
)
return GenAIOperation.CHAT
def resolve_output_type(call_type: str | None) -> GenAIOutputType | None:
"""Map a litellm ``call_type`` to a ``gen_ai.output.type`` value, or ``None``
for a route that doesn't pin the output modality."""
if not call_type:
return None
return _OUTPUT_TYPE_BY_CALL_TYPE.get(call_type.lower())

2
litellm/integrations/otel/plumbing/providers.py
View file

@ -436,6 +436,8 @@ def build_tracer_provider(
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
exp = _exporter_from_spec(spec)
provider.add_span_processor(
_processor_for(

29
litellm/integrations/otel/plumbing/routing.py
View file

@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import (
get_tracer,
)
from litellm.integrations.otel.presets import (
dynamic_otlp_endpoint,
dynamic_otlp_headers,
project_routing_headers,
)
@ -129,7 +130,9 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
# Oldest-first so an overflow of draining providers sheds the stalest.
self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers
@ -182,12 +185,16 @@ class TenantTracerCache:
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
cache_key: Final = (
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -200,15 +207,16 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems],
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
self._providers[cache_key] = built
return built
@ -257,6 +265,7 @@ class TenantTracerCache:
self,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -272,7 +281,8 @@ class TenantTracerCache:
``Authorization``), which must survive routing to a project.
"""
exporters: Final = [
self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
@ -281,6 +291,7 @@ class TenantTracerCache:
spec: ExporterSpec,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> ExporterSpec:
kind: Final = spec.kind.lower()
if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@ -291,4 +302,10 @@ class TenantTracerCache:
if project_headers and kind not in _GRPC_KINDS
else base
)
return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
update: Final = { # mutable-ok: model_copy(update=...) requires a plain dict
field: value
for field, value in (("headers", routed), ("endpoint", endpoint))
if (field == "headers" and routed != spec.headers)
or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
}
return spec if not update else spec.model_copy(update=update)

70
litellm/integrations/otel/presets/__init__.py
View file

@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
)
from litellm.integrations.otel.presets.langtrace import langtrace_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.newrelic import (
newrelic_dynamic_endpoint,
newrelic_dynamic_headers,
newrelic_preset,
)
from litellm.integrations.otel.presets.phoenix import (
phoenix_preset,
phoenix_project_headers,
@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
#: registered value matches the preset interface.
PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"weave_otel": weave_preset,
}
PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
{
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"newrelic": newrelic_preset,
"weave_otel": weave_preset,
}
)
#: Callback name → per-request OTLP header builder (team/key multi-tenant
#: routing). Only integrations that support dynamic credentials appear here —
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
#: default tracer.
DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
MappingProxyType(
{
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"newrelic": newrelic_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
)
)
#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
#: destination host varies per tenant (from a fixed region table, never a
#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
#: authoritative.
DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
MappingProxyType(
{
"newrelic": newrelic_dynamic_endpoint,
}
)
)
#: Callback name → per-request *routing* header builder, sourced from the key/team
@ -98,17 +123,34 @@ def project_routing_headers(
return builder(auth_metadata)
def dynamic_otlp_endpoint(
callback_name: str | None,
dynamic_params: StandardCallbackDynamicParams | None,
) -> str | None:
"""Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
``None`` means "keep the preset's own endpoint".
"""
resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
if resolver is None or not dynamic_params:
return None
return resolver(dynamic_params)
__all__ = [
"DYNAMIC_ENDPOINT_BY_CALLBACK",
"DYNAMIC_HEADERS_BY_CALLBACK",
"PRESET_BY_CALLBACK",
"PROJECT_HEADERS_BY_CALLBACK",
"Preset",
"agentops_preset",
"arize_preset",
"dynamic_otlp_endpoint",
"dynamic_otlp_headers",
"langfuse_preset",
"langtrace_preset",
"levo_preset",
"newrelic_preset",
"phoenix_preset",
"project_routing_headers",
"weave_preset",

104
litellm/integrations/otel/presets/newrelic.py Normal file
View file

@ -0,0 +1,104 @@
"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import (
ExporterOwner,
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
#: region enum rather than a free-form endpoint, so callback vars can never
#: redirect telemetry to an arbitrary host.
NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
{
"us": "https://otlp.nr-data.net",
"eu": "https://otlp.eu01.nr-data.net",
}
)
_DEFAULT_REGION: Final = "us"
class _NewRelicSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
# The same env vars the agent-based integration documents; the key is the
# operator-level fallback for traffic without team credentials, the region
# picks that fallback's data center, and the record-content flag keeps its
# documented meaning when the OTel path replaces the agent.
license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
def newrelic_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
settings: Final = _NewRelicSettings()
base: Final = config_overrides or OpenTelemetryV2Config()
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
(settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
)
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,
headers=(f"api-key={settings.license_key}" if settings.license_key else None),
owner=ExporterOwner.NEWRELIC,
requires_headers=True,
),
],
# New Relic ingests the OTLP GenAI semantic conventions natively.
"mapper_names": ensure_mappers(base.mapper_names, "genai"),
**(
{"capture_message_content": ("span_only" if settings.record_content else "no_content")}
if settings.record_content is not None
else {}
),
}
)
def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request New Relic OTLP headers from team/key dynamic params."""
api_key: Final = params.get("newrelic_api_key")
return {header: value for header, value in (("api-key", api_key),) if value}
def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
"""Per-request OTLP endpoint for the team's ``newrelic_region``.
Always the team's own region endpoint, defaulting to US when the team left
the region unset. It never falls through to the preset's endpoint, which
follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
ingest key must not inherit the operator's region and have its US-account
spans rejected by an EU-configured default (or vice versa). An unknown
region likewise resolves to the documented US default rather than a guess.
"""
region: Final = params.get("newrelic_region")
default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
if not region:
return default_endpoint
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
region,
", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
)
return default_endpoint
return endpoint

27
litellm/integrations/prometheus.py
View file

@ -215,7 +215,9 @@ class PrometheusLogger(CustomLogger):
# request latency metrics
self.litellm_request_total_latency_metric = self._histogram_factory(
"litellm_request_total_latency_metric",
"Total latency (seconds) for a request to LiteLLM",
"End-to-end latency (seconds) for a request to LiteLLM Proxy Server, from the moment "
"the request reached the proxy through the end of processing -- includes "
"authentication, pre-call hooks, the LLM API call, and post-call processing",
labelnames=self.get_labels_for_metric("litellm_request_total_latency_metric"),
buckets=self.latency_buckets,
)
@ -458,7 +460,8 @@ class PrometheusLogger(CustomLogger):
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time spent in request queue before processing starts (seconds)",
"Time (seconds) from request arrival at the proxy to the start of pre-call "
"processing -- includes authentication and any ASGI-level queueing",
labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
buckets=self.latency_buckets,
)
@ -2078,27 +2081,37 @@ class PrometheusLogger(CustomLogger):
_labels,
)
# total request latency
# request queue time (time from arrival to processing start) -- read first so
# it can be folded into the total-latency metric below. start_time/end_time
# only span from after auth completes, so without this the "total" latency
# metric silently excludes auth and pre-call hook time.
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
# total request latency: true end-to-end, from request arrival (queue_time_seconds,
# when available) through the end of processing.
total_time_seconds: Final = self._safe_duration_seconds(
start_time=start_time,
end_time=end_time,
)
if total_time_seconds is not None:
_observed_total_time_seconds: Final = (
total_time_seconds + queue_time_seconds
if queue_time_seconds is not None and queue_time_seconds >= 0
else total_time_seconds
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_request_total_latency_metric.labels(**_labels).observe(total_time_seconds)
self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
if queue_time_seconds is not None and queue_time_seconds >= 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_queue_time_seconds"),

99
litellm/integrations/s3_v2.py
View file

@ -11,6 +11,7 @@ import time
from collections.abc import Mapping
from datetime import datetime
from typing import Final, cast
from urllib.parse import quote
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
)
def _build_object_url(self, s3_object_key: str) -> str:
"""
Build the exact URL that is both signed and sent, with the key percent-encoded once.
S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
the two signatures disagree (403 SignatureDoesNotMatch).
"""
encoded_key: Final = quote(s3_object_key, safe="/")
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
def _sse_headers(self) -> Mapping[str, str]:
candidates: Final = {
"x-amz-server-side-encryption": self.s3_server_side_encryption,
@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
aws_region_name=self.s3_region_name,
)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
httpx_client: Final = _get_httpx_client(
params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
)
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
try:
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
url: Final = self._build_object_url(s3_object_key)
# Prepare the request for GET operation
# For GET requests, we need x-amz-content-sha256 with hash of empty string
@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
headers: Final = {
"x-amz-content-sha256": empty_string_hash,
}
req: Final = requests.Request("GET", url, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
request_url: Final = prepped.url or url
response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
if response.status_code != 200:
verbose_logger.exception("S3 object not found, saw response=", response.text)

53
litellm/integrations/websearch_interception/ARCHITECTURE.md
View file

@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
---
## Loop Ceiling
One intercepted request can chain several follow-up model calls, since the model often searches again after
reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
```yaml
litellm_settings:
websearch_interception_params:
enabled_providers: ["bedrock"]
max_agentic_loops: 5
```
Or per deployment, which wins over the feature-level setting:
```yaml
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
max_agentic_loops: 5
```
Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
carries it is ignored and one request can never drive an unbounded number of upstream model calls.
Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
validator down there would drop the deployment silently instead of refusing to start.
When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
rather than as a replacement.
Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
Both are tracked separately
---
## Streaming Support
WebSearch interception works transparently with both streaming and non-streaming requests.

22
litellm/integrations/websearch_interception/handler.py
View file

@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
self,
enabled_providers: list[LlmProviders | str] | None = None,
search_tool_name: str | None = None,
max_agentic_loops: int | None = None,
):
"""
Args:
@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
Default: None (all providers enabled)
search_tool_name: Name of search tool configured in router's search_tools.
If None, will attempt to use first available search tool.
max_agentic_loops: How many follow-up model calls one intercepted request
may chain before the loop is refused and the turn ends.
If None, LiteLLM's default of 3 applies.
"""
super().__init__()
# Convert enum values to strings for comparison
@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
else:
self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
self.search_tool_name = search_tool_name
self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
self._request_has_websearch = False # Track if current request has web search
@staticmethod
def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
"""
Reject loop ceilings the agentic loop cannot honor, at config load time.
"""
return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
async def try_short_circuit_search(
self,
model: str,
@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: "my-perplexity-search"
max_agentic_loops: 5
Usage:
config = litellm_settings.get("websearch_interception_params", {})
@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# Extract parameters from config
enabled_providers_str: Final = config.get("enabled_providers", None)
search_tool_name: Final = config.get("search_tool_name", None)
max_agentic_loops: Final = config.get("max_agentic_loops", None)
# Convert string provider names to LlmProviders enum values
enabled_providers: list[LlmProviders | str] | None = None
@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return cls(
enabled_providers=enabled_providers,
search_tool_name=search_tool_name,
max_agentic_loops=max_agentic_loops,
)
@staticmethod
@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag

313
litellm/interactions/background_cost_polling.py Normal file
View file

@ -0,0 +1,313 @@
"""
Cost tracking for background interactions.
A create request with ``background=true`` returns ``in_progress`` with no
usage block, and GET polls are deliberately never billed (billing them would
double-charge every poll; the GET response also does not echo ``background``,
so a poll cannot be told apart from a re-fetch of an already-billed
interaction). The create call is therefore the only place that can own
billing: it schedules a poll task that fetches the interaction until it
reaches a terminal status and logs the final usage as a single success event
attributed to the original request.
``requires_action`` is terminal for the interaction it names. The API has no
operation that resumes one: a caller answers a tool request by creating a new
interaction whose ``previous_interaction_id`` points at it, and that new
interaction bills itself. The paused interaction keeps the tokens it already
spent producing the tool request, so it is billed and settled where it stops
rather than polled until the timeout, which would both lose that usage and
hold its budget reservation open for the whole timeout window.
Deleting an interaction makes every subsequent poll fail, which would let a
caller retrieve the completed output themselves and then delete it before the
poll task settles, leaving the work unbilled and the budget reservation
refunded at the poll timeout. ``adelete`` therefore settles any pending poll
for the interaction before dispatching the delete: it fetches the current
state with the create's credentials, bills it if it is terminal with usage,
and releases the reservation otherwise. A settlement gate on the create's
logging object makes the poll task and the delete path mutually exclusive, so
the interaction is billed exactly once no matter who settles first.
"""
import asyncio
from collections.abc import Awaitable, Callable, Iterator, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_TERMINAL_STATUSES: Final = frozenset(
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
)
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
@dataclass(frozen=True, slots=True)
class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
**{
"no-log": True
}, # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
)
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0
interval = initial
while interval > 0 and elapsed + interval <= timeout:
yield interval
elapsed += interval
interval = min(interval * 2, maximum)
_SETTLED_KEY = "background_interaction_settled"
def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
return logging_obj.model_call_details.get(_SETTLED_KEY) is True
def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
"""
Exactly-once gate between the poll task and the delete-time settlement:
both run on the same event loop and neither awaits between reading and
setting the flag, so whichever claims first owns billing or release.
"""
if _is_settled(logging_obj):
return False
logging_obj.model_call_details[_SETTLED_KEY] = True # rebind-ok: both settlers must see the same settlement flag
return True
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
last_seen_status: str | None = None
for interval in _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
await asyncio.sleep(interval)
if _is_settled(context.logging_obj):
return
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
e,
)
continue
last_seen_status = response.status
if response.status not in _TERMINAL_STATUSES:
continue
if not _claim_settlement(context.logging_obj):
return
if response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
else:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
"the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
"will not be tracked",
context.interaction_id,
context.timeout_seconds,
last_seen_status,
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
"""
The proxy keeps the pre-call budget reservation open for an in-progress
background interaction so concurrent creates cannot stack past the budget.
The completion success event reconciles it to the actual cost; when the
interaction terminates without billable usage (or polling gives up, or it
is deleted before settling), no such event fires, so whoever claims the
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
budget_reservation = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
try:
await release_budget_reservation(budget_reservation=budget_reservation)
except Exception: # noqa: BLE001 # a failed release must not crash the poll task; counters expire via TTL
verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
"""
Claiming the settlement makes the claimer solely responsible for the
reservation, and no one retries a claim that is already set. A billing
failure here must therefore release the reservation on its way out, or it
stays pinned at the estimated cost until the whole poll times out.
"""
try:
await logging_obj.async_log_background_interaction_completion(result=response)
except Exception:
await _release_open_budget_reservation(logging_obj=logging_obj)
raise
def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
"""
The single gate deciding whether a create's response gets a poll task.
The proxy's success callback defers releasing the budget reservation for
exactly these responses, on the promise that a poll task will settle them,
so a response one site accepts and the other refuses strands its
reservation on the spend counters with nothing left to reconcile it.
``queued`` belongs here alongside ``in_progress``. It is the API's
not-started-yet state, so it reaches a terminal status the same way and
needs polling for the same reason: nothing else in the proxy ever bills a
create that came back without usage, so a status missing from both this
set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
"""
return response.status in _POLLABLE_STATUSES and bool(response.id)
def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
"""
Whether a response arriving with no usage block is a normal outcome rather
than lost billing data. An interaction that is still running, or that
stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
has nothing to charge for and should not raise a cost-tracking alarm.
``completed`` and ``requires_action`` both mean the model produced output,
so a usage block is always expected with them. If one arrives without it
the charge for real work has been lost, which is precisely what the
proxy's cost-tracking alert exists to surface.
"""
return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
@dataclass(frozen=True, slots=True)
class _ActiveBackgroundPoll:
task: "asyncio.Task[None]"
context: BackgroundInteractionPollContext
_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {} # mutable-ok: asyncio needs strong refs to running poll tasks
def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is not None and entry.task is task:
del _ACTIVE_POLLS[interaction_id]
def maybe_schedule_background_interaction_cost_polling(
response: object,
create_kwargs: Mapping[str, object],
custom_llm_provider: str,
) -> "asyncio.Task[None] | None":
from litellm.litellm_core_utils.litellm_logging import Logging
if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
return None
if not isinstance(response, InteractionsAPIResponse):
return None
if not is_pollable_background_interaction(response):
return None
logging_obj = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
try:
asyncio.get_running_loop()
except RuntimeError:
return None
api_key = create_kwargs.get("api_key")
api_base = create_kwargs.get("api_base")
context = BackgroundInteractionPollContext(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
_ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
task.add_done_callback(
lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
)
return task
async def maybe_settle_background_interaction_before_delete(
interaction_id: str,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is None:
return
context = entry.context
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation
verbose_logger.debug(
"Could not fetch background interaction %s before delete, releasing its reservation: %s",
interaction_id,
e,
)
if _claim_settlement(context.logging_obj):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
return
await _release_open_budget_reservation(logging_obj=context.logging_obj)

12
litellm/interactions/main.py
View file

@ -40,6 +40,10 @@ from typing import Any, Final
import httpx
import litellm
from litellm.interactions.background_cost_polling import (
maybe_schedule_background_interaction_cost_polling,
maybe_settle_background_interaction_before_delete,
)
from litellm.interactions.http_handler import interactions_http_handler
from litellm.interactions.utils import (
InteractionsAPIRequestUtils,
@ -171,6 +175,12 @@ async def acreate(
else:
response = init_response
maybe_schedule_background_interaction_cost_polling(
response=response,
create_kwargs=kwargs,
custom_llm_provider=custom_llm_provider,
)
return response
except Exception as e:
raise litellm.exception_type(
@ -462,6 +472,8 @@ async def adelete(
loop: Final = asyncio.get_event_loop()
kwargs["adelete_interaction"] = True
await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id)
func: Final = partial(
delete,
interaction_id=interaction_id,

59
litellm/litellm_core_utils/agentic_loop_settings.py Normal file
View file

@ -0,0 +1,59 @@
"""
Shared validation for the agentic loop ceiling.
``max_agentic_loops`` can be set in two places, and the two disagreed about
what a bad value means. The feature-level
``litellm_settings.websearch_interception_params.max_agentic_loops`` was
checked at config load, while a per-deployment
``model_list[].litellm_params.max_agentic_loops`` was passed straight through
to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
turning the tightest ceiling into the loosest one, and let a per-deployment
``"three"`` boot the proxy and then fail every request to that model.
Both settings now go through :func:`validated_max_agentic_loops`, which names
the field it rejected so the error says which line of the config to fix.
Anything that spells a whole number is still accepted, because the old
``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
"""
from typing import Final
DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
def _as_whole_number(value: object) -> int | None:
"""
Return ``value`` as an int when it spells a whole number, else ``None``.
``bool`` is excluded explicitly because it is an ``int`` subclass, so
``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
"""
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value) if value.is_integer() else None
if isinstance(value, str):
try:
return int(value.strip())
except ValueError:
return None
return None
def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
"""
Return ``max_agentic_loops`` as an int, or raise naming ``field``.
"""
if max_agentic_loops is None:
return None
ceiling: Final = _as_whole_number(max_agentic_loops)
if ceiling is None:
raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
if ceiling < 1:
raise ValueError(f"{field} must be at least 1, got {ceiling}")
return ceiling

9
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -5,6 +5,10 @@ from typing import Final, cast
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
return depth, max_loops, fingerprints

1
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -2301,6 +2301,7 @@ def exception_type(
or custom_llm_provider == "custom_openai"
or custom_llm_provider in litellm.openai_compatible_providers
or custom_llm_provider == "mistral"
or custom_llm_provider == "runwayml"
):
_map_openai_exception(
model=model,

24
litellm/litellm_core_utils/get_provider_specific_headers.py
View file

@ -1,3 +1,4 @@
from collections.abc import Sequence
from typing import Final
from litellm.types.utils import ProviderSpecificHeader
@ -6,13 +7,17 @@ from litellm.types.utils import ProviderSpecificHeader
class ProviderSpecificHeaderUtils:
@staticmethod
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | None,
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
custom_llm_provider: str | None,
) -> dict:
"""
Get the provider specific headers for the given custom llm provider.
Supports comma-separated provider lists for headers that work across multiple providers.
Accepts either a single ProviderSpecificHeader or a sequence of them. Each entry
carries its own comma-separated provider list, so headers that are safe for several
providers and headers that are safe for exactly one can travel on the same request
without sharing a scope. Entries whose provider list does not contain
`custom_llm_provider` contribute nothing.
Returns:
Dict: The provider specific headers for the given custom llm provider
@ -20,10 +25,15 @@ class ProviderSpecificHeaderUtils:
if provider_specific_header is None or custom_llm_provider is None:
return {}
stored_providers: Final = provider_specific_header.get("custom_llm_provider", "")
provider_list: Final = [p.strip() for p in stored_providers.split(",")]
scoped_headers: Final = (
(provider_specific_header,) if isinstance(provider_specific_header, dict) else provider_specific_header
)
if custom_llm_provider in provider_list:
return provider_specific_header.get("extra_headers", {})
matched_headers: Final = {}
for scoped_header in scoped_headers:
stored_providers = scoped_header.get("custom_llm_provider", "")
provider_list = [p.strip() for p in stored_providers.split(",")]
if custom_llm_provider in provider_list:
matched_headers.update(scoped_header.get("extra_headers", {}))
return {}
return matched_headers

32
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final = [
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
@ -72,8 +72,10 @@ _supported_callback_params: Final = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
"turn_off_message_logging",
]
)
_request_blocked_callback_params: Final = frozenset(
{
@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset(
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
}
)
# Request-blocked params that must still reach ``standard_callback_dynamic_params``
# when the proxy itself stamped them from admin-configured team/key callback
# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads
# ``standard_callback_dynamic_params``, so without this overlay a blocked param
# could never drive routing at all.
_trusted_overlay_callback_params: Final = frozenset(
{
"newrelic_api_key",
"newrelic_region",
}
)
@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params(
if param in kwargs:
_param_value = kwargs.get(param)
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params(
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for param, trusted_value in get_trusted_callback_params(kwargs):
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
return standard_callback_dynamic_params

202
litellm/litellm_core_utils/litellm_logging.py
View file

@ -71,6 +71,9 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
)
from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.redact_messages import (
@ -83,6 +86,10 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
@ -2145,6 +2152,11 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, OpenAIModerationResponse)
or isinstance(logging_result, OCRResponse) # OCR
or isinstance(logging_result, SearchResponse) # Search API
or (
isinstance(logging_result, InteractionsAPIResponse)
and logging_result.usage is not None
and self._is_interactions_create_call_type()
)
or isinstance(logging_result, dict)
and logging_result.get("object") == "vector_store.search_results.page"
or isinstance(logging_result, dict)
@ -2157,6 +2169,87 @@ class Logging(LiteLLMLoggingBaseClass):
return True
return False
def _is_interactions_create_call_type(self) -> bool:
"""
Only interaction creation is billable. GET polls, deletes, and cancels
also return an ``InteractionsAPIResponse`` (with usage once completed),
so recognizing those would write spend on every poll of a background
interaction. The proxy sets ``call_type`` from its route_type
(``create_interaction``/``acreate_interaction``); the SDK sets it from
the decorated function name (``create``/``acreate``).
Recognition additionally requires a usage block (checked at the call
site): a ``background=true`` create returns ``in_progress`` without
usage, and billing it would write a $0 spend log under the interaction
id that collides with the row the background poll task writes once the
interaction completes (see
``litellm.interactions.background_cost_polling``).
"""
return self.call_type in (
CallTypes.create_interaction.value,
CallTypes.acreate_interaction.value,
"create",
"acreate",
)
async def async_log_background_interaction_completion(
self,
result: InteractionsAPIResponse,
) -> None:
"""
Log the terminal result of a background interaction as a fresh success
event. The create request already ran success logging for its
``in_progress`` response (no usage, so no cost was tracked); clearing
the dedup flags lets the completed result flow through cost calculation
and spend tracking exactly once, spanning create to completion.
The poll fetched this body through its own client call, which priced it
against a throwaway logging object holding none of this request's
deployment context: no ``model_info``, no router ``model_id``, no
deployment ``litellm_params``. Keeping that price would bill a
custom-priced deployment at the wrong rate, and it would also satisfy
the "already calculated" shortcut and skip repricing here, leaving the
cost breakdown at the zeros the usage-less create stamped and writing
those zeros to the spend log. Dropping it makes this event price the
settled body itself, against the deployment that served the create.
The same throwaway call stamped the deployment identity that travels
with the price, so ``model_id`` and ``litellm_model_name`` go with it.
Left in place they overwrite the create's real deployment with the
poll's empty one in the payload every logging integration reads.
"""
settled_hidden_params: Final = getattr(result, "_hidden_params", None)
if isinstance(settled_hidden_params, dict):
for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
settled_hidden_params.pop(poll_scoped_key, None)
self._reset_success_emission_dedupe()
await self.async_success_handler(result=result)
def _reset_success_emission_dedupe(self) -> None:
"""
Success callbacks dedupe per request, because the sync and async
handlers both fire on some paths and would otherwise report one call
twice. A settled background interaction is a genuinely second success
event on the same request, so every such marker has to be cleared or
the completion, the only event that carries usage and cost, is
discarded as a duplicate of the in-progress create.
"""
self.model_call_details.pop("has_logged_async_success", None)
litellm_params = self.model_call_details.get("litellm_params")
if not isinstance(litellm_params, dict):
return
metadata = litellm_params.get("metadata")
if not isinstance(metadata, dict):
return
otel_internal = metadata.get("_otel_internal")
if not isinstance(otel_internal, dict):
return
spans_logged = otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):
return
for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]:
del spans_logged[scope]
def _flush_passthrough_collected_chunks_helper(
self,
raw_bytes: list[bytes],
@ -2282,7 +2375,9 @@ class Logging(LiteLLMLoggingBaseClass):
is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None = None
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
) = None
if "complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response = self._get_assembled_streaming_response(
@ -2768,14 +2863,14 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None] = (
self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
if complete_streaming_response is not None:
@ -3558,7 +3653,7 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
@ -3583,9 +3678,40 @@ class Logging(LiteLLMLoggingBaseClass):
),
)
return result.response
elif isinstance(result, InteractionsAPIStreamingResponse):
return self._assemble_completed_interaction_response(result)
else:
return None
@staticmethod
def _assemble_completed_interaction_response(
result: InteractionsAPIStreamingResponse,
) -> InteractionsAPIResponse | None:
"""
The Interactions API streaming iterator hands the terminal event to the
success handlers: the new schema (Api-Revision: 2026-05-20) emits
``interaction.completed`` carrying the full interaction object, the
legacy schema (2026-05-07) emits a chunk with ``status="completed"``
and usage on the chunk itself. Build the equivalent non-streaming
response so cost calculation and spend tracking see one shape.
"""
if result.event_type == "interaction.completed" and result.interaction is not None:
return InteractionsAPIResponse(**result.interaction)
if result.status == "completed":
return InteractionsAPIResponse(
**result.model_dump(
exclude={ # mutable-ok: pydantic types exclude as set[str], which a frozenset does not satisfy
"event_type",
"delta",
"index",
"step",
"interaction_id",
"interaction",
}
)
)
return None
def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
"""
Handles logging for Anthropic messages responses.
@ -4503,6 +4629,9 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
for callback in _in_memory_loggers:
if isinstance(callback, NewRelicLogger):
return callback
@ -4789,7 +4918,11 @@ def get_custom_logger_compatible_class(
if isinstance(callback, SMTPEmailLogger):
return callback
elif logging_integration == "newrelic":
from litellm.integrations.otel.logger import OpenTelemetryV2
for callback in _in_memory_loggers:
if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic":
return callback
if isinstance(callback, NewRelicLogger):
return callback
return None
@ -5085,6 +5218,8 @@ class StandardLoggingPayloadSetup:
elif isinstance(usage, dict):
if ResponseAPILoggingUtils._is_response_api_usage(usage):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
return Usage(**usage)
raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
@ -5111,6 +5246,8 @@ class StandardLoggingPayloadSetup:
if isinstance(_raw, dict):
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
return _raw
if isinstance(_raw, Usage):
return _raw.model_dump()
@ -5615,6 +5752,37 @@ def _extract_response_obj_and_hidden_params(
return response_obj, hidden_params
def _autorouter_savings_for_payload(
request_metadata: Mapping[str, object],
model: str | None,
custom_llm_provider: str | None,
model_id: str | None,
usage_object: Mapping[str, object] | None,
cost_breakdown: Mapping[str, object] | None,
) -> float | None:
"""The auto-router savings figure for the payload, or ``None`` when there is none.
Lazy proxy import: the savings module lives with the spend trackers that own the
math, and SDK-only installs have no proxy package to import.
"""
try:
from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload
except Exception: # noqa: BLE001 # SDK-only install: no savings driver to run
return None
try:
return autorouter_savings_for_logging_payload(
request_metadata=request_metadata,
model=model,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
usage_object=usage_object,
cost_breakdown=cost_breakdown,
)
except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging
verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
return None
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: Any | BaseModel | dict,
@ -5772,6 +5940,16 @@ def get_standard_logging_object_payload(
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
autorouter_savings: Final = _autorouter_savings_for_payload(
request_metadata=metadata,
model=model_name,
custom_llm_provider=custom_llm_provider,
model_id=_model_id,
usage_object=usage_dict,
cost_breakdown=request_cost_breakdown,
)
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
id=str(id),
litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
@ -5802,7 +5980,8 @@ def get_standard_logging_object_payload(
metadata=clean_metadata,
cache_key=clean_hidden_params["cache_key"],
response_cost=response_cost,
cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,
total_tokens=usage_dict.get("total_tokens", 0),
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
@ -5998,6 +6177,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
call_type="completion",
stream=False,
response_cost=response_cost,
autorouter_savings=None,
response_cost_failure_debug_info=None,
status="success",
total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT),

127
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,9 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
TranscriptionUsageDurationObject,
TranscriptionUsageTokensObject,
@ -34,3 +37,127 @@ class TranscriptionUsageObjectTransformation:
),
)
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
"image": "image_tokens",
"video": "video_tokens",
"document": "text_tokens",
}
)
def _modality_field(entry: Mapping[str, Any]) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
def _token_count(value: object) -> int:
return value if isinstance(value, int) else 0
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
for field in fields
}
)
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
)
def _subtract_cached_from_input(
input_sums: Mapping[str, int],
cached_sums: Mapping[str, int],
total_cached_tokens: int,
) -> Mapping[str, int]:
if cached_sums:
return MappingProxyType(
{field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()}
)
if total_cached_tokens and "text_tokens" in input_sums:
return MappingProxyType(
{
**input_sums,
"text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens),
}
)
return input_sums
class InteractionsUsageObjectTransformation:
"""
Maps the Google Interactions API usage block (total_input_tokens,
output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the
generic cost calculator and spend tracking can bill it.
"""
@staticmethod
def is_interactions_usage_object(usage_object: object) -> bool:
if not isinstance(usage_object, dict):
return False
if "prompt_tokens" in usage_object or "input_tokens" in usage_object:
return False
return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
**input_sums,
)
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,
)
if output_sums or reasoning_tokens
else None
)
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
prompt_tokens_details=prompt_tokens_details,
completion_tokens_details=completion_tokens_details,
cache_read_input_tokens=total_cached_tokens or None,
)

80
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,8 +1,88 @@
from collections.abc import Mapping
from typing import Final
import litellm
def _form_field_value(value: object) -> str:
if value is True:
return "true"
if value is False:
return "false"
return str(value)
def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
if isinstance(value, Mapping):
return tuple(
item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def _is_form_scalar(value: object) -> bool:
return value is not None and not isinstance(value, (Mapping, list, tuple))
def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
if isinstance(value, Mapping):
return tuple(
item
for subkey, subvalue in value.items()
for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
if all(_is_form_scalar(entry) for entry in value):
serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry)))
return ((key, serialized_fields),) if serialized_fields else ()
return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
"""
Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed
multipart body, applying ``sources`` in order so a later source wins on a key collision
under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK
serializes them, so provider params reach a multipart request without handing the httpx
encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a
single field carrying a tuple value, which httpx emits as one repeated part per element, so
every element survives instead of collapsing to the last under ``dict.update``.
"""
return tuple(
pair
for source in sources
if source is not None
for top_key, top_value in source.items()
for pair in _flatten_form_data_field(top_key, top_value)
)
def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]:
"""
Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less
request is still sent as multipart/form-data, working around httpx downgrading a
file-less ``data=`` payload to application/x-www-form-urlencoded.
"""
return tuple(
(key, (None, serialized))
for top_key, top_value in data.items()
for key, serialized in _flatten_form_field(top_key, top_value)
)
def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None:
"""
Ensure that the extra_body sent in the request is safe, otherwise users will see this error

11
litellm/litellm_core_utils/logging_utils.py
View file

@ -3,6 +3,7 @@ import functools
import inspect
import re
import time
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
@ -268,6 +269,16 @@ def _set_duration_in_model_call_details(
verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e)
def speech_request_body(model: str, voice: str, optional_params: Mapping[str, object]) -> Mapping[str, object]:
"""Speech request body for telemetry, without the caller headers the provider SDKs
take as request kwargs rather than body fields."""
return { # mutable-ok: loggers isinstance-check the request body as a dict
"model": model,
"voice": voice,
**{key: value for key, value in optional_params.items() if key != "extra_headers"},
}
def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.

36
litellm/litellm_core_utils/logging_worker.py
View file

@ -5,7 +5,7 @@ import asyncio
import atexit
import contextvars
import logging
from collections.abc import Coroutine
from collections.abc import Coroutine, Iterator
from typing import Final
from typing_extensions import TypedDict
@ -61,6 +61,19 @@ class LoggingWorker:
# Register cleanup handler to flush remaining events on exit
atexit.register(self._flush_on_exit)
@staticmethod
def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
"""Pop every task still queued, without awaiting them, so they can be moved to another queue."""
def _pop_until_empty() -> Iterator[LoggingTask]:
while True:
try:
yield queue.get_nowait()
except asyncio.QueueEmpty:
return
return tuple(_pop_until_empty())
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist or if event loop has changed."""
try:
@ -69,14 +82,27 @@ class LoggingWorker:
# No running loop, can't initialize
return
# Check if we need to reinitialize due to event loop change
# The queue, semaphore and worker task are all bound to the loop that created them. On a
# loop change we hand the still-pending tasks to a fresh queue instead of dropping them,
# so queued spend-logging coroutines are not silently discarded (and never left un-awaited).
if self._queue is not None and self._bound_loop is not current_loop:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
# Clear old state - these are bound to the old loop
self._queue = None
carried_over: Final = self._drain_pending(self._queue)
new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
for carried_task in carried_over:
new_queue.put_nowait(carried_task)
if carried_over:
verbose_logger.warning(
"LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
len(carried_over),
)
else:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
self._sem = None
self._worker_task = None
self._running_tasks.clear()
self._queue = new_queue
self._bound_loop = current_loop
return
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)

58
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -28,8 +28,12 @@ from litellm.types.llms.openai import (
ChatCompletionAssistantMessage,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
ChatCompletionRedactedThinkingBlock,
ChatCompletionResponseMessage,
ChatCompletionTextObject,
ChatCompletionThinkingBlock,
ChatCompletionToolParam,
ChatCompletionUserMessage,
)
@ -466,6 +470,8 @@ def update_messages_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
for message in messages:
@ -504,6 +510,11 @@ def update_messages_with_model_file_ids(
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
file_object_file_field["format"] = format
@ -531,6 +542,8 @@ def update_responses_input_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
if isinstance(input, str):
@ -574,6 +587,13 @@ def update_responses_input_with_model_file_ids(
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)
else:
# Not a managed file, keep as-is
updated_content.append(content_item)
@ -1549,6 +1569,44 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
return None, message_content
def _readable_thinking_text(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
) -> str:
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
if block.get("type") != "thinking":
return ""
thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag
return str(thinking or "")
def reasoning_content_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> str:
"""Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
Redacted blocks carry no readable text, so they contribute nothing.
"""
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
def responses_reasoning_item_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> ChatCompletionReasoningItem | None:
"""Build a Responses API `reasoning` input item from Anthropic thinking blocks.
The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
did not mint itself, while an item without an id is always accepted.
"""
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
for block in thinking_blocks
if (text := _readable_thinking_text(block))
]
if not summary:
return None
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
def _parse_content_for_reasoning(
message_text: str | None,
) -> tuple[str | None, str | None]:

8
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -124,6 +124,14 @@ def ptu_identity_error(
return None
PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to")
def declares_ptu(model_info: Mapping[str, object]) -> bool:
"""Whether any PTU field is set here, including one too malformed to charge."""
return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS)
def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
"""Why this PTU configuration cannot be honoured, else None.

33
litellm/llms/anthropic/chat/transformation.py
View file

@ -1,7 +1,8 @@
import json
import re
import time
from collections.abc import Mapping, Sequence
from collections.abc import Callable, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
import httpx
@ -121,6 +122,32 @@ else:
# response side.
_ANTHROPIC_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
_ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[Any], bool]]] = MappingProxyType(
{
"null": lambda v: v is None,
"boolean": lambda v: isinstance(v, bool),
"integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
"number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"string": lambda v: isinstance(v, str),
"array": lambda v: isinstance(v, list),
"object": lambda v: isinstance(v, dict),
}
)
def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool:
"""Whether ``schema``'s ``enum`` cannot match its declared ``type``."""
enum_values: Final = schema.get("enum")
declared_type: Final = schema.get("type")
if not isinstance(enum_values, list) or declared_type is None:
return False
if isinstance(declared_type, list):
return True
check: Final = _ENUM_TYPE_CHECKS.get(declared_type)
return check is not None and not all(check(value) for value in enum_values)
# Single, internal-only key on ``litellm_params`` used to thread the per-
# request reverse map (sanitized -> original) from request build to response
# parsing. ``litellm_params`` is never serialized to a provider; ``optional_
@ -565,9 +592,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
else:
result["description"] = constraint_note
drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
for key, value in schema.items():
if key in unsupported_fields:
continue
if key == "type" and drops_conflicting_type:
continue
if key == "description" and "description" in result:
# Already handled above
continue

10
litellm/llms/anthropic/common_utils.py
View file

@ -440,6 +440,16 @@ class AnthropicModelInfo(BaseLLMModelInfo):
"""
return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
@staticmethod
def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool:
"""Whether ``model`` is an adaptive-thinking model that still accepts legacy
``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family).
The model cost map is authoritative: an explicit ``supports_legacy_thinking``
entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations``
rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape.
"""
return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider)
@staticmethod
def maybe_drop_disabled_thinking(
model: str,

11
litellm/llms/anthropic/experimental_pass_through/adapters/handler.py
View file

@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
)
from litellm.llms.anthropic.experimental_pass_through.utils import (
is_reasoning_auto_summary_enabled,
local_model_name,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
except Exception:
pass
if isinstance(model, str) and model and not model.startswith("responses/"):
# Prefix model with "responses/" to route to OpenAI Responses API
completion_kwargs["model"] = f"responses/{model}"
if isinstance(model, str) and model and "responses/" not in model:
local_model: Final = model.removeprefix(f"{custom_llm_provider}/")
completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}"
auto_summary: Final = is_reasoning_auto_summary_enabled()
@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=True,
@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
if stream:
transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
completion_response,
model=model,
model=local_model_name(model, kwargs.get("custom_llm_provider")),
tool_name_mapping=tool_name_mapping,
polyfill_result=polyfill_result,
is_async=False,

4
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -64,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.prompt_templates.factory import (
@ -592,6 +593,9 @@ class LiteLLMAnthropicMessagesAdapter:
assistant_message["tool_calls"] = tool_calls
if len(thinking_blocks) > 0:
assistant_message["thinking_blocks"] = thinking_blocks
reasoning_content = reasoning_content_from_thinking_blocks(thinking_blocks)
if reasoning_content:
assistant_message["reasoning_content"] = reasoning_content
new_messages.append(assistant_message)
return new_messages

8
litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py
View file

@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
else:
passthrough_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": block_dict,
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode())
content_block_stop: Final = {"type": "content_block_stop", "index": index}
chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode())
return chunks

41
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response
_RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool:
"""Return True when the provider should use the Responses API path.
def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
from litellm.main import responses_api_bridge_check
model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
return model_info.get("mode") == "responses"
def _responses_mode_is_lost_by_prefix_strip(
requested_model: str, resolved_model: str, custom_llm_provider: str
) -> bool:
"""Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
"""
if requested_model == resolved_model:
return False
return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
resolved_model, custom_llm_provider
)
def _should_route_to_responses_api(
custom_llm_provider: str | None,
requested_model: str | None = None,
resolved_model: str | None = None,
) -> bool:
"""Return True when the request should use the Responses API path.
Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
opt out and route OpenAI/Azure requests through chat/completions instead.
"""
if litellm.use_chat_completions_url_for_anthropic_messages:
return False
return custom_llm_provider in _RESPONSES_API_PROVIDERS
if custom_llm_provider in _RESPONSES_API_PROVIDERS:
return True
if custom_llm_provider is None or requested_model is None or resolved_model is None:
return False
return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
@ -533,7 +564,7 @@ def anthropic_messages_handler(
_shared_kwargs: Final = dict(
max_tokens=max_tokens,
messages=messages,
model=model,
model=original_model,
metadata=metadata,
stop_sequences=stop_sequences,
stream=stream,
@ -551,7 +582,7 @@ def anthropic_messages_handler(
custom_llm_provider=custom_llm_provider,
**kwargs,
)
if _should_route_to_responses_api(custom_llm_provider):
if _should_route_to_responses_api(custom_llm_provider, original_model, model):
return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs)
# The in-gateway context_management polyfill runs inside

10
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
def _translate_legacy_thinking_for_adaptive_model(
model: str, optional_params: dict, custom_llm_provider: str
) -> None:
"""Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
Caller-provided ``output_config.effort`` is never overridden.
"""Translate legacy ``thinking.type=enabled`` to adaptive for the
adaptive-thinking models that reject it (4.7+ and the 5 families).
Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
legacy shape natively, so it is forwarded verbatim and the caller's
``budget_tokens`` cap keeps applying. Caller-provided
``output_config.effort`` is never overridden.
"""
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
return
if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
return
thinking: Final = optional_params.get("thinking")
if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
return

11
litellm/llms/anthropic/experimental_pass_through/messages/utils.py
View file

@ -44,6 +44,7 @@ class AnthropicMessagesRequestUtils:
filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None}
if model is not None:
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
AnthropicConfig._maybe_drop_speed_param(
model=model,
@ -51,6 +52,16 @@ class AnthropicMessagesRequestUtils:
drop_params=drop_params,
custom_llm_provider=custom_llm_provider,
)
for param in ("temperature", "top_p", "top_k"):
if param in filtered_params:
AnthropicModelInfo._apply_sampling_param( # pyright: ignore[reportPrivateUsage] # same gating the /chat/completions path applies; forking it would drift
optional_params=filtered_params,
model=model,
param=param,
value=filtered_params.pop(param),
drop_params=drop_params,
output_key=param,
)
return cast(AnthropicMessagesRequestOptionalParams, filtered_params)

9
litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py
View file

@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
)
from litellm.types.llms.openai import ResponsesAPIResponse
from ..utils import local_model_name
from .streaming_iterator import AnthropicResponsesStreamWrapper
from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
result: Final = await litellm.aresponses(**responses_kwargs)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):
@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
result: Final = litellm.responses(**responses_kwargs)
if stream:
wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
wrapper: Final = AnthropicResponsesStreamWrapper(
responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
)
return wrapper.async_anthropic_sse_wrapper()
if not isinstance(result, ResponsesAPIResponse):

5
litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py
View file

@ -152,7 +152,10 @@ class AnthropicResponsesStreamWrapper:
if block_idx < 0:
if not delta:
return
block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""})
block_idx = self._open_block(
item_id,
{"type": "thinking", "thinking": "", "signature": ""}, # mutable-ok: API message payload
)
self._chunk_queue.append(
{
"type": "content_block_delta",

112
litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
View file

@ -6,12 +6,14 @@ path used for OpenAI and Azure models.
"""
import json
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from itertools import groupby
from typing import Any, Final, cast
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
responses_reasoning_item_from_thinking_blocks,
with_prompt_cache_breakpoint,
)
from litellm.litellm_core_utils.reasoning_effort_utils import (
@ -36,7 +38,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
AnthropicUsage,
)
from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
from litellm.types.llms.openai import (
ChatCompletionThinkingBlock,
ResponseAPIUsage,
ResponsesAPIResponse,
)
class LiteLLMAnthropicToResponsesAPIAdapter:
@ -100,6 +106,58 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text")) # pyright: ignore[reportUnnecessaryIsInstance] # untrusted client payload
]
@staticmethod
def _summary_part_text(part: object) -> str:
if isinstance(part, Mapping):
mapping: Final = cast(Mapping[str, Any], part) # cast-ok: summary parts are untyped provider json
return str(mapping.get("text") or "")
return str(getattr(part, "text", None) or "")
@classmethod
def _thinking_blocks_from_reasoning_item(
cls,
summary: Iterable[object],
) -> tuple[dict[str, Any], ...]: # mutable-ok: API message payload
"""Anthropic thinking blocks for one Responses reasoning item.
The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
value would be replayed as a real one and rejected by every backend that verifies it.
"""
return tuple(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
for part in summary
if (text := cls._summary_part_text(part))
)
@staticmethod
def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str:
"""Group a run of consecutive thinking blocks together; keep every other block alone."""
index, block = indexed_block
return "thinking" if block.get("type") == "thinking" else f"block:{index}"
@classmethod
def _assistant_group_to_input_item(
cls, group: tuple[Mapping[str, Any], ...]
) -> dict[str, Any] | None: # mutable-ok: API message payload
first: Final = group[0]
btype: Final = first.get("type")
if btype == "thinking":
blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group) # cast-ok: untrusted client payload
reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return None if reasoning_item is None else dict(reasoning_item) # mutable-ok: API message payload
if btype == "tool_use":
return { # mutable-ok: API message payload
"type": "function_call",
"call_id": first.get("id", ""),
"name": first.get("name", ""),
"arguments": json.dumps(first.get("input", {})), # mutable-ok: API message payload
}
return None
def translate_messages_to_responses_input(
self,
messages: list[AllAnthropicPassThroughMessageValues],
@ -113,6 +171,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
user image -> message(role=user, input_image)
user tool_result -> function_call_output
assistant text -> message(role=assistant, output_text)
assistant thinking -> reasoning
assistant tool_use -> function_call
"""
input_items: Final[list[dict[str, Any]]] = []
@ -233,27 +292,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
}
)
elif isinstance(content, list):
asst_parts: list[dict[str, Any]] = []
for block in content:
if not isinstance(block, dict):
continue
btype = block.get("type")
if btype == "text":
asst_parts.append({"type": "output_text", "text": block.get("text", "")})
elif btype == "tool_use":
# tool_use becomes a top-level function_call item
input_items.append(
{
"type": "function_call",
"call_id": block.get("id", ""),
"name": block.get("name", ""),
"arguments": json.dumps(block.get("input", {})),
}
)
elif btype == "thinking":
thinking_text = block.get("thinking", "")
if thinking_text:
asst_parts.append({"type": "output_text", "text": thinking_text})
blocks = tuple(block for block in content if isinstance(block, dict))
input_items.extend(
item
for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
)
asst_parts: list[dict[str, Any]] = [ # mutable-ok: API message payload
{"type": "output_text", "text": block.get("text", "")} # mutable-ok: API message payload
for block in blocks
if block.get("type") == "text"
]
if asst_parts:
input_items.append(
{
@ -514,16 +563,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
for item in response.output:
if isinstance(item, ResponseReasoningItem):
for summary in item.summary:
text = getattr(summary, "text", "")
if text:
content.append(
AnthropicResponseContentBlockThinking(
type="thinking",
thinking=text,
signature=None,
).model_dump()
)
content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
elif isinstance(item, ResponseOutputMessage):
for part in item.content:
@ -555,6 +595,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
content.append(
AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
)
elif item_type == "reasoning":
content.extend(
self._thinking_blocks_from_reasoning_item(
cast(Iterable[object], item.get("summary") or ()), # cast-ok: untyped provider json
)
)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))

5
litellm/llms/anthropic/experimental_pass_through/utils.py
View file

@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
def local_model_name(model: str, custom_llm_provider: object) -> str:
"""The id the provider itself knows, for reporting back to the caller in ``message_start``."""
return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model
def is_reasoning_auto_summary_enabled() -> bool:
"""Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"

23
litellm/llms/azure/azure.py
View file

@ -17,7 +17,7 @@ from openai import (
import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
@ -1352,6 +1352,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
organization: str | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
azure_ad_token: str | None = None,
azure_ad_token_provider: Callable | None = None,
aspeech: bool | None = None,
@ -1373,6 +1374,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
azure_ad_token_provider=azure_ad_token_provider,
max_retries=max_retries,
timeout=timeout,
logging_obj=logging_obj,
client=client,
litellm_params=litellm_params,
)
@ -1387,6 +1389,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
litellm_params=litellm_params,
)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(azure_client.base_url),
},
)
response: Final = azure_client.audio.speech.create(
model=model,
voice=voice,
@ -1408,6 +1419,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
azure_ad_token_provider: Callable | None,
max_retries: int,
timeout: float | httpx.Timeout,
logging_obj: LiteLLMLoggingObj,
client=None,
litellm_params: dict | None = None,
) -> HttpxBinaryResponseContent:
@ -1421,6 +1433,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
litellm_params=litellm_params,
)
logging_obj.pre_call(
input=input,
api_key=api_key,
additional_args={ # mutable-ok: loggers isinstance-check this payload as a dict
"complete_input_dict": speech_request_body(model, voice, optional_params),
"api_base": str(azure_client.base_url),
},
)
azure_response: Final = await azure_client.audio.speech.create(
model=model,
voice=voice,

29
litellm/llms/azure/common_utils.py
View file

@ -3,6 +3,7 @@ import hashlib
import json
import os
from collections.abc import Callable, Mapping
from functools import lru_cache
from typing import Any, Final, Literal, NamedTuple, cast
import httpx
@ -75,6 +76,24 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
return {**llm_response_headers, **openai_headers}
@lru_cache(maxsize=128)
def _cached_entra_id_token_provider(
tenant_id: str,
client_id: str,
client_secret: str,
scope: str,
) -> Callable[[], str]:
"""Build (once per credential set) a bearer token provider backed by a `ClientSecretCredential`.
The credential caches the access token internally and only talks to Entra ID when it is close
to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
per request.
"""
from azure.identity import ClientSecretCredential, get_bearer_token_provider
return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
def get_azure_ad_token_from_entra_id(
tenant_id: str,
client_id: str,
@ -93,8 +112,6 @@ def get_azure_ad_token_from_entra_id(
Returns:
callable that returns a bearer token.
"""
from azure.identity import ClientSecretCredential, get_bearer_token_provider
verbose_logger.debug("Getting Azure AD Token from Entra ID")
if tenant_id.startswith("os.environ/"):
@ -120,9 +137,13 @@ def get_azure_ad_token_from_entra_id(
)
if _tenant_id is None or _client_id is None or _client_secret is None:
raise ValueError("tenant_id, client_id, and client_secret must be provided")
credential: Final = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
token_provider: Final = get_bearer_token_provider(credential, scope)
token_provider: Final = _cached_entra_id_token_provider(
tenant_id=_tenant_id,
client_id=_client_id,
client_secret=_client_secret,
scope=scope,
)
verbose_logger.debug("token_provider %s", token_provider)

3
litellm/llms/azure/search/__init__.py Normal file
View file

@ -0,0 +1,3 @@
from litellm.llms.azure.search.transformation import BingGroundingSearchConfig
__all__ = ("BingGroundingSearchConfig",)

442
litellm/llms/azure/search/transformation.py Normal file
View file

@ -0,0 +1,442 @@
"""
Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search`
tool to search the web (Grounding with Bing Search).
Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding
Setup:
1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g.
https://<account>.services.ai.azure.com/api/projects/<project>
2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1);
it runs the grounded search and its tokens are billed on that deployment
3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search
project connection id to use the `bing_grounding` tool; without it the
project's built-in `web_search` tool is used
4. Auth: pass api_key (an Azure API key, sent in the api-key header), or set
BING_GROUNDING_TOKEN to an Entra bearer token for scope
https://ai.azure.com/.default, or configure azure-identity (AZURE_CLIENT_ID /
AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any
DefaultAzureCredential source) and the token is minted automatically
Usage:
response = litellm.search(
query="latest AI developments",
search_provider="bing_grounding",
max_results=5,
)
"""
from __future__ import annotations
from collections.abc import Callable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
import httpx
from pydantic import BaseModel, ConfigDict, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
SearchResponse,
SearchResult,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding"
PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT"
MODEL_ENV: Final = "BING_GROUNDING_MODEL"
CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID"
TOKEN_ENV: Final = "BING_GROUNDING_TOKEN"
ENTRA_SCOPE: Final = "https://ai.azure.com/.default"
_RESPONSES_PATH: Final = "/openai/v1/responses"
_SNIPPET_FALLBACK_LENGTH: Final = 300
_UPSTREAM_ERROR_STATUS: Final = 502
_RESPONSE_COST_HEADER: Final = "llm_provider-x-litellm-response-cost"
class _Annotation(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
url: str | None = None
title: str | None = None
start_index: int | None = None
end_index: int | None = None
class _ContentPart(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
text: str = ""
annotations: tuple[_Annotation, ...] = ()
class _OutputItem(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
type: str = ""
content: tuple[_ContentPart, ...] = ()
class _ErrorBody(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
message: str | None = None
class _IncompleteDetails(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
reason: str | None = None
class _ResponsesEnvelope(BaseModel):
"""A Foundry Responses API body. `output` is required: a body without it is not a
Responses API response and must not be reported as a successful empty search.
A 200 body can still carry `status` `failed` or `incomplete`; those are surfaced as
errors rather than reported as a successful empty search."""
model_config = ConfigDict(extra="ignore", frozen=True)
output: tuple[_OutputItem, ...]
status: str | None = None
error: _ErrorBody | None = None
incomplete_details: _IncompleteDetails | None = None
class _ErrorEnvelope(BaseModel):
model_config = ConfigDict(extra="ignore", frozen=True)
error: _ErrorBody | None = None
def _unwrap_error_detail(error_message: str) -> str:
"""
Surface the human-readable message inside Foundry's error envelope.
Tool failures nest a second JSON document as a string inside `error.message`
(observed live for `bing_grounding` connection errors), so the unwrap runs twice.
Falls back to the raw body for anything else.
"""
try:
envelope: Final = _ErrorEnvelope.model_validate_json(error_message)
except ValidationError:
return error_message
message: Final = envelope.error.message if envelope.error else None
if message is None:
return error_message
try:
nested: Final = _ErrorBody.model_validate_json(message)
except ValidationError:
return message
return nested.message or message
def _snippet(text: str, annotation: _Annotation) -> str:
"""
The text a citation supports, not the citation marker itself.
A url_citation's start/end indices span the inline marker ("([host](url))"),
which follows the claim it backs, so the snippet is the marker's own line up
to where the marker starts.
"""
start: Final = annotation.start_index
marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text)
claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip()
if claim:
return claim[-_SNIPPET_FALLBACK_LENGTH:]
return text[:_SNIPPET_FALLBACK_LENGTH]
def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]:
"""One result per cited URL: first occurrence wins, order preserved as answered."""
cited: Final = tuple(
SearchResult(
title=annotation.title or "",
url=annotation.url or "",
snippet=_snippet(part.text, annotation),
date=None,
last_updated=None,
)
for item in envelope.output
if item.type == "message"
for part in item.content
if part.type == "output_text"
for annotation in part.annotations
if annotation.type == "url_citation" and annotation.url
)
first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)})
return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited))
def _valid_max_results(max_results: object) -> int | None:
"""A positive-int `max_results`, else None. Rejects bools, an `int` subclass, and
non-positive values so neither the request-side `count` nor the response-side cap
forwards a value the other would silently ignore.
"""
if isinstance(max_results, bool) or not isinstance(max_results, int):
return None
return max_results if max_results > 0 else None
def _requested_max_results(response_kwargs: Mapping[str, object]) -> int | None:
"""The unified `max_results` cap the caller asked for, if any.
The built-in web_search tool has no server-side result-count knob, so the cap is
enforced here after the fact; connection mode also honors it as a hard ceiling on
top of the tool's `count` hint.
"""
optional_params: Final = response_kwargs.get("optional_params")
if not isinstance(optional_params, Mapping):
return None
return _valid_max_results(optional_params.get("max_results"))
def _capped(results: tuple[SearchResult, ...], max_results: int | None) -> tuple[SearchResult, ...]:
return results[:max_results] if max_results is not None else results
class _SearchConfiguration(BaseModel):
model_config = ConfigDict(frozen=True)
project_connection_id: str
count: int | None = None
class _BingGroundingParams(BaseModel):
model_config = ConfigDict(frozen=True)
search_configurations: tuple[_SearchConfiguration, ...]
class _BingGroundingTool(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["bing_grounding"] = "bing_grounding"
bing_grounding: _BingGroundingParams
class _UserLocation(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["approximate"] = "approximate"
country: str
class _WebSearchTool(BaseModel):
model_config = ConfigDict(frozen=True)
type: Literal["web_search"] = "web_search"
user_location: _UserLocation | None = None
class _ResponsesRequest(BaseModel):
model_config = ConfigDict(frozen=True)
model: str
input: str
tools: tuple[_BingGroundingTool | _WebSearchTool, ...]
def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool:
connection_id: Final = get_secret_str(CONNECTION_ID_ENV)
max_results: Final = optional_params.get("max_results")
country: Final = optional_params.get("country")
if connection_id:
configuration: Final = _SearchConfiguration(
project_connection_id=connection_id,
count=_valid_max_results(max_results),
)
return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,)))
location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None
return _WebSearchTool(user_location=location)
def _default_entra_token_minter() -> str:
from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
class BingGroundingSearchConfig(BaseSearchConfig):
def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None:
super().__init__()
self._entra_token_minter = entra_token_minter
@staticmethod
def ui_friendly_name() -> str:
return "Grounding with Bing Search"
def validate_environment(
self,
headers: dict[str, str], # mutable-ok: BaseSearchConfig.validate_environment signature
api_key: str | None = None,
api_base: str | None = None,
**kwargs: object, # kwargs-ok: BaseSearchConfig.validate_environment signature
) -> dict[str, str]: # mutable-ok: the http handler passes this straight to httpx as headers
"""
Validate environment and return headers.
Returns a new dict rather than mutating ``headers``: the http handler calls this
a second time after ``litellm/search/main.py`` already did, so it has to be idempotent.
"""
return { # mutable-ok: httpx requires a plain dict of headers
**headers,
**self._auth_header(api_key, api_base),
"Content-Type": "application/json",
}
def _auth_header(self, api_key: str | None, api_base: str | None) -> Mapping[str, str]:
"""
A caller-supplied ``api_key`` is an Azure API key and rides the ``api-key`` header;
an Entra bearer token (``BING_GROUNDING_TOKEN`` or one minted via azure-identity)
rides ``Authorization: Bearer``. Foundry rejects the wrong scheme for each.
"""
if api_key:
return MappingProxyType({"api-key": api_key})
token: Final = self.resolve_server_api_key(
caller_api_key=None,
caller_api_base=api_base,
key_env_vars=(TOKEN_ENV,),
base_env_var=PROJECT_ENDPOINT_ENV,
default_api_base=None,
) or self._mint_entra_token(api_base)
return MappingProxyType({"Authorization": f"Bearer {token}"})
def _mint_entra_token(self, caller_api_base: str | None) -> str:
self._assert_trusted_api_base_for_server_credential(
caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token"
)
minter: Final = self._entra_token_minter or _default_entra_token_minter
try:
return minter()
except Exception as e:
raise ValueError(
f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "
f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) "
f"for scope {ENTRA_SCOPE}. Underlying error: {e}"
) from e
def get_complete_url(
self,
api_base: str | None,
optional_params: dict[str, object], # mutable-ok: BaseSearchConfig.get_complete_url signature
data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature
**kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature
) -> str:
resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV)
if not resolved_base:
raise ValueError(
f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
)
trimmed: Final = resolved_base.rstrip("/")
if trimmed.endswith(_RESPONSES_PATH):
return trimmed
return f"{trimmed}{_RESPONSES_PATH}"
def transform_search_request(
self,
query: str | list[str], # mutable-ok: BaseSearchConfig.transform_search_request signature
optional_params: dict[str, object], # mutable-ok: base signature
**kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_request signature
) -> dict[str, object]: # mutable-ok: the http handler passes this straight to httpx as the JSON body
"""
Transform Search request to the Foundry Responses API format.
The unified params map as far as the API allows:
- max_results -> the bing_grounding search configuration's `count`; the built-in
web_search tool has no result-count knob, so that mode instead caps the returned
results after the fact (see transform_search_response)
- country -> web_search's approximate `user_location` (bing_grounding's `market`
wants a full locale like en-US, which a bare country code cannot fill)
- search_domain_filter, max_tokens_per_page -> no API equivalent, dropped
"""
model: Final = get_secret_str(MODEL_ENV)
if not model:
raise ValueError(
f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project "
f"that runs the grounded search, e.g. gpt-4.1."
)
request: Final = _ResponsesRequest(
model=model,
input=" ".join(query) if isinstance(query, list) else query,
tools=(_search_tool(optional_params),),
)
return request.model_dump(mode="json", exclude_none=True)
def transform_search_response(
self,
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
**kwargs: object, # kwargs-ok: BaseSearchConfig.transform_search_response signature
) -> SearchResponse:
try:
parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content)
except ValidationError as e:
raise self.get_error_class(
error_message=f"response does not match the Foundry Responses API schema: {e}",
status_code=raw_response.status_code,
headers=dict(raw_response.headers), # mutable-ok: BaseSearchConfig.get_error_class signature
)
if parsed.status == "failed":
detail: Final = (
parsed.error.message if parsed.error and parsed.error.message else "the grounded search failed"
)
raise self._upstream_error(detail, raw_response)
results: Final = _capped(_citation_results(parsed), _requested_max_results(kwargs))
if not results and parsed.status == "incomplete":
reason: Final = (
parsed.incomplete_details.reason
if parsed.incomplete_details and parsed.incomplete_details.reason
else "unknown reason"
)
raise self._upstream_error(f"the grounded search was incomplete: {reason}", raw_response)
return self._priced(results)
def _upstream_error(self, detail: str, raw_response: httpx.Response) -> Exception:
return self.get_error_class(
error_message=detail,
status_code=_UPSTREAM_ERROR_STATUS,
headers=dict(raw_response.headers), # mutable-ok: BaseSearchConfig.get_error_class signature
)
def _priced(self, results: tuple[SearchResult, ...]) -> SearchResponse:
"""web_search mode runs no paid Grounding with Bing transaction, so it must not
inherit the connection-mode ``bing_grounding/search`` price; zero its per-query
cost while leaving connection mode to the cost map."""
response: Final = SearchResponse(
results=list(results), # mutable-ok: SearchResponse.results is list[SearchResult]
object="search",
)
if get_secret_str(CONNECTION_ID_ENV):
return response
response._hidden_params[
"additional_headers"
] = { # mutable-ok: response_cost_calculator writes into _hidden_params
_RESPONSE_COST_HEADER: 0.0
}
return response
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict[str, str], # mutable-ok: BaseSearchConfig.get_error_class signature
) -> Exception:
detail: Final = _unwrap_error_detail(error_message).rstrip(". ")
return BaseLLMException(
status_code=status_code,
message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.",
headers=headers,
)

10
litellm/llms/azure_ai/chat/transformation.py
View file

@ -30,7 +30,12 @@ class AzureFoundryErrorStrings(str, enum.Enum):
SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control")
NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
"thinking_blocks",
"reasoning_content",
"provider_specific_fields",
"cache_control",
)
class AzureAIStudioConfig(OpenAIConfig):
@ -173,7 +178,8 @@ class AzureAIStudioConfig(OpenAIConfig):
"""
- Azure AI Studio doesn't support content as a list. This handles:
1. Strips message fields that are not part of the OpenAI chat-completions
schema (thinking_blocks, provider_specific_fields, cache_control).
schema (thinking_blocks, reasoning_content, provider_specific_fields,
cache_control).
Azure AI Foundry backends set additionalProperties=false and reject
these with "Extra inputs are not permitted", which breaks multi-turn
Anthropic-format clients that echo thinking blocks back as history.

47
litellm/llms/azure_ai/common_utils.py
View file

@ -1,9 +1,54 @@
from collections.abc import Mapping
from typing import Final, Literal
import litellm
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
"""
Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
Accepts the same credential set as the `azure` provider: service principal
(`tenant_id` / `client_id` / `client_secret`), a pre-fetched `azure_ad_token`, an OIDC
federated token, username/password, or `DefaultAzureCredential` / managed identity.
"""
from litellm.llms.azure.common_utils import get_azure_ad_token
params = GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
return get_azure_ad_token(params)
def get_azure_ai_auth_headers(
api_key: str | None,
litellm_params: Mapping[str, object] | None = None,
api_key_header: AzureAIApiKeyHeader = "Authorization",
api_key_env_var: str = "AZURE_AI_API_KEY",
) -> Mapping[str, str]:
"""
Build the auth headers for an Azure AI Foundry route.
Prefers the API key when one is configured, and otherwise falls back to Entra ID / OAuth,
sending the access token as a bearer token.
"""
if api_key:
return {api_key_header: f"Bearer {api_key}" if api_key_header == "Authorization" else api_key}
azure_ad_token = get_azure_ai_entra_token(litellm_params=litellm_params)
if azure_ad_token:
return {"Authorization": f"Bearer {azure_ad_token}"}
raise ValueError(
f"Missing Azure AI credentials - set an API key (`api_key` or {api_key_env_var}), or Entra ID / OAuth "
"credentials (`tenant_id` + `client_id` + `client_secret`, `azure_ad_token`, an OIDC token, or a managed "
"identity with `litellm.enable_azure_ad_token_refresh = True`)"
)
class AzureFoundryModelInfo(BaseLLMModelInfo):
@ -43,7 +88,7 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
@staticmethod
def get_api_key(api_key: str | None = None) -> str | None:
return api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY")
return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY")
@property
def api_version(self, api_version: str | None = None) -> str | None:

18
litellm/llms/azure_ai/image_edit/flux2_transformation.py
View file

@ -5,7 +5,10 @@ from typing import Any, Final
from httpx._types import RequestFiles
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.azure_ai.image_generation.flux_transformation import (
AzureFoundryFluxImageGenerationConfig,
)
@ -71,16 +74,13 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
"""
Validate Azure AI Foundry environment and set up authentication
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key,
**get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="Api-Key",
),
"Content-Type": "application/json",
}
)

19
litellm/llms/azure_ai/image_edit/mai_transformation.py
View file

@ -3,7 +3,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.azure_ai.image_generation.mai_transformation import (
AzureFoundryMAIImageGenerationConfig,
)
@ -91,15 +94,13 @@ class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig):
litellm_params: dict | None = None,
api_base: str | None = None,
) -> dict:
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. "
"Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
headers.update(
get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="api-key",
)
headers.update({"api-key": api_key})
)
return headers
def get_complete_url(

22
litellm/llms/azure_ai/image_edit/transformation.py
View file

@ -3,7 +3,10 @@ from typing import Final
import httpx
import litellm
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
from litellm.secret_managers.main import get_secret_str
from litellm.utils import _add_path_to_api_base
@ -30,19 +33,14 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
) -> dict:
"""
Validate Azure AI Foundry environment and set up authentication
Uses Api-Key header format
Uses the Api-Key header format, or an Entra ID / OAuth bearer token when no key is set
"""
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if not api_key:
raise ValueError(
f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
)
headers.update(
{
"Api-Key": api_key, # Azure AI Foundry uses Api-Key header format
}
get_azure_ai_auth_headers(
api_key=AzureFoundryModelInfo.get_api_key(api_key),
litellm_params=litellm_params,
api_key_header="Api-Key",
)
)
return headers

16
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -26,6 +26,7 @@ from litellm.constants import (
)
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import (
OCR_REQUEST_FORMAT_PARAM,
BaseOCRConfig,
@ -236,17 +237,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"""
Validate environment and return headers for Azure Document Intelligence.
Authentication uses Ocp-Apim-Subscription-Key header.
Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer
token when no subscription key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR)
if api_key is None:
raise ValueError(
"Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter"
)
# Validate API base/endpoint is provided
if api_base is None:
api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
@ -257,7 +254,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
)
headers = {
"Ocp-Apim-Subscription-Key": api_key,
**get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params,
api_key_header="Ocp-Apim-Subscription-Key",
api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR,
),
"Content-Type": "application/json",
**headers,
}

10
litellm/llms/azure_ai/ocr/transformation.py
View file

@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_convert_url_to_base64,
convert_url_to_base64,
)
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
from litellm.secret_managers.main import get_secret_str
@ -47,17 +48,12 @@ class AzureAIOCRConfig(MistralOCRConfig):
"""
Validate environment and return headers for Azure AI OCR.
Azure AI uses Bearer token authentication with AZURE_AI_API_KEY.
Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set.
"""
# Get API key from environment if not provided
if api_key is None:
api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR)
if api_key is None:
raise ValueError(
"Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params"
)
# Validate API base is provided
if api_base is None:
api_base = get_secret_str("AZURE_AI_API_BASE")
@ -68,7 +64,7 @@ class AzureAIOCRConfig(MistralOCRConfig):
)
headers = {
"Authorization": f"Bearer {api_key}",
**get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
"Content-Type": "application/json",
**headers,
}

8
litellm/llms/azure_ai/rerank/transformation.py
View file

@ -2,12 +2,14 @@
Translate between Cohere's `/rerank` format and Azure AI's `/rerank` format.
"""
from collections.abc import Mapping
from typing import Final
import httpx
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
from litellm.llms.cohere.rerank.transformation import CohereRerankConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import RerankResponse
@ -64,15 +66,13 @@ class AzureAIRerankConfig(CohereRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key
if api_key is None:
raise ValueError("Azure AI API key is required. Please set 'AZURE_AI_API_KEY' or 'litellm.azure_key'")
default_headers: Final = {
"Authorization": f"Bearer {api_key}",
**get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
"accept": "application/json",
"content-type": "application/json",
}

9
litellm/llms/base_llm/files/transformation.py
View file

@ -11,9 +11,9 @@ from litellm.types.llms.openai import (
AllMessageValues,
CreateFileRequest,
FileContentRequest,
FileListPage,
OpenAICreateFileRequestOptionalParams,
OpenAIFileObject,
OpenAIFilesPurpose,
)
from litellm.types.utils import LlmProviders, ModelResponse
@ -240,10 +240,13 @@ class BaseFileEndpoints(ABC):
@abstractmethod
async def afile_list(
self,
purpose: OpenAIFilesPurpose | None,
purpose: str | None,
litellm_parent_otel_span: Span | None,
user_api_key_dict: UserAPIKeyAuth,
limit: int | None = None,
after: str | None = None,
**data: dict,
) -> list[OpenAIFileObject]:
) -> FileListPage:
pass
@abstractmethod

2
litellm/llms/base_llm/rerank/transformation.py
View file

@ -1,4 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
import httpx
@ -24,6 +25,7 @@ class BaseRerankConfig(ABC):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
pass

21
litellm/llms/base_llm/videos/transformation.py
View file

@ -1,9 +1,10 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
import httpx
from httpx._types import RequestFiles
from httpx._types import FileContent, RequestFiles
from litellm.types.responses.main import *
from litellm.types.router import GenericLiteLLMParams
@ -91,6 +92,14 @@ class BaseVideoConfig(ABC):
raise ValueError("api_base is required")
return api_base
def use_multipart_form_data(self) -> bool:
"""
Whether video create requests without files must still be sent as
multipart/form-data (the encoding the OpenAI SDK always uses for
/videos), instead of falling back to JSON.
"""
return False
@abstractmethod
def transform_video_create_request(
self,
@ -332,14 +341,18 @@ class BaseVideoConfig(ABC):
api_base: str,
litellm_params: GenericLiteLLMParams,
headers: dict,
video_file: FileContent | None = None,
extra_body: dict[str, Any] | None = None,
prefetched_source_data: dict[str, Any] | None = None,
) -> tuple[str, dict]:
) -> tuple[str, Mapping[str, object], RequestFiles | None]:
"""
Transform the video edit request into a URL and JSON data.
Transform the video edit request into a URL plus either JSON data or
multipart form fields and files.
Returns:
Tuple[str, Dict]: (url, data) for the POST request
tuple[str, Mapping[str, object], RequestFiles | None]: (url, data,
files). When files is None the handler sends data as JSON; otherwise
data holds the form fields and files holds the uploaded source video.
"""
raise NotImplementedError("video edit is not supported for this provider")

8
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -567,6 +567,10 @@ class AWSEventStreamDecoder:
if thinking_blocks:
self._thinking_ran = True
carries_message_content: Final = any(
key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason", "trace")
)
model_response_provider_specific_fields: Final = {}
if "trace" in chunk_data:
trace: Final = chunk_data.get("trace")
@ -577,8 +581,8 @@ class AWSEventStreamDecoder:
finish_reason=finish_reason,
index=0, # Always 0 - Bedrock never returns multiple choices
delta=Delta(
content=text,
role="assistant",
content=text if carries_message_content else None,
role="assistant" if carries_message_content else None,
tool_calls=[tool_use] if tool_use else None,
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
thinking_blocks=thinking_blocks,

2
litellm/llms/cohere/rerank/transformation.py
View file

@ -1,3 +1,4 @@
from collections.abc import Mapping
from typing import Any, Final
import httpx
@ -81,6 +82,7 @@ class CohereRerankConfig(BaseRerankConfig):
model: str,
api_key: str | None = None,
optional_params: dict | None = None,
litellm_params: Mapping[str, object] | None = None,
) -> dict:
if api_key is None:
api_key = get_secret_str("COHERE_API_KEY") or get_secret_str("CO_API_KEY") or litellm.cohere_key

Some files were not shown because too many files have changed in this diff Show more