scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_cli_keyring_bounded_calls
Some checks failed
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details

Browse source
This commit is contained in:
mateo 2026-08-26 16:40:08 +00:00
commit 5ea9261e1f
2671 changed files with 75187 additions and 26258 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

195
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -651,126 +651,6 @@ jobs:
- auth_ui_unit_tests_coverage.xml
- auth_ui_unit_tests_coverage
proxy_behavior_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy management behavior tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_behavior \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
proxy_security_tests:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Seed DB schema via prisma db push
command: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Run proxy security tests
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_security_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
schema_migration_check:
docker:
- *python312_image
- image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
environment:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
working_directory: ~/project
environment:
# An empty database; the test applies every committed migration itself.
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Generate Prisma Client
command: uv run --no-sync python -m prisma generate
- run:
name: Check schema.prisma is in sync with committed migrations
command: |
mkdir -p test-results
uv run --no-sync python -m pytest tests/proxy_migration_tests \
-v --junitxml=test-results/junit.xml --durations=10
no_output_timeout: 15m
- store_test_results:
path: test-results
litellm_router_testing: # Runs all tests with the "router" keyword
docker:
- *python312_image
@ -858,7 +738,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -985,7 +865,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=20 \
-n 4 \
@ -1030,7 +910,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1074,7 +954,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2 \
@ -1120,7 +1000,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
--retries 3 --retry-delay 5"
@ -1211,7 +1091,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1255,7 +1135,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1274,40 +1154,6 @@ jobs:
paths:
- search_coverage.xml
- search_coverage
litellm_mapped_enterprise_tests:
docker:
- *python312_image
working_directory: ~/project
resource_class: large
steps:
- checkout
- skip_if_unrelated_changes
- setup_google_dns
- install_uv
- install_rust
- run:
name: Install Dependencies
command: |
uv sync --frozen --all-groups --all-extras --python 3.12
- setup_litellm_enterprise_pip
- run:
name: Run enterprise tests
command: |
uv run --no-sync python -m prisma generate
mkdir -p test-results
TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
echo "$TEST_FILES" | circleci tests run \
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \
--junitxml=test-results/junit-enterprise.xml \
--durations=10 \
-n 4"
no_output_timeout: 15m
# Store test results
- store_test_results:
path: test-results
batches_testing:
docker:
- *python312_image
@ -1333,7 +1179,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1377,7 +1223,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 2"
@ -1422,7 +1268,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 \
-n 4"
@ -1501,7 +1347,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \
--junitxml=test-results/junit.xml \
--durations=5 \
@ -1546,7 +1392,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5"
no_output_timeout: 15m
@ -1599,7 +1445,7 @@ jobs:
--verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \
--cov=./litellm --cov-report=xml \
--cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \
--durations=5 -n 2 \
--reruns 2 --reruns-delay 1"
@ -3105,12 +2951,6 @@ workflows:
filters: *main_branches
- auth_ui_unit_tests:
filters: *main_branches
- proxy_behavior_tests:
filters: *main_branches
- proxy_security_tests:
filters: *main_branches
- schema_migration_check:
filters: *main_branches
- build_docker_database_image:
filters: *main_branches
- e2e_ui_testing:
@ -3167,8 +3007,6 @@ workflows:
filters: *main_branches
- search_testing:
filters: *main_branches
- litellm_mapped_enterprise_tests:
filters: *main_branches
- batches_testing:
filters: *main_branches
- litellm_utils_testing:
@ -3191,7 +3029,6 @@ workflows:
- guardrails_testing
- ocr_testing
- search_testing
- litellm_mapped_enterprise_tests
- batches_testing
- litellm_utils_testing
- pass_through_unit_testing

9
.github/CODEOWNERS vendored
View file

@ -1,5 +1,10 @@
/ui/ @yuneng-jiang @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
/ui/ @yuneng-berri @ryan-crabbe-berri
/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
/ui/Dockerfile
/ui/nginx.conf
/ui/litellm-dashboard/src/lib/http/schema.d.ts
/ui/litellm-dashboard/tsconfig.tsbuildinfo
/model_prices_and_context_window.json @mateo-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri
/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
/.github/CODEOWNERS @yuneng-berri

31
.github/actions/cache-cargo-build/action.yml vendored Normal file
View file

@ -0,0 +1,31 @@
name: "Cache the Rust build"
description: >-
Cache the Cargo registry and target directory the root package's build needs,
so only the first job on a given Cargo.lock compiles the bridge from scratch.
litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
in release mode before it can produce a wheel. `uv sync` therefore pays a full
build in every job that installs the workspace: measured at 2m40s per unit shard
on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
it, because the uv cache holds wheels uv downloads rather than wheels it builds,
and a path dependency whose source moves every commit could never hit that cache
anyway. Cargo rebuilds only what changed when its target directory survives, so a
warm job pays for the bridge crate alone.
The key namespace is separate from test-rust.yml's. Both cache the same directory,
but that workflow fills it with debug and clippy artifacts, which a release build
cannot reuse, and a shared key would let whichever ran first deny the other a save.
runs:
using: composite
steps:
- name: Restore the Cargo registry and target directory
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-cargo-release-

46
.github/ci-coverage-allowlist.yml vendored
View file

@ -5,24 +5,21 @@ description: >-
test_paths:
- reason: >-
The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
2026-08-20 by collecting the directory under each job's own selector: 118 tests across
these eight files are selected by none of them. Listed so the gap is a decision rather
than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
when tests/local_testing is ported off CircleCI, where the keyless part of this suite
belongs in a real job
What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
split that porting tests/local_testing off CircleCI will force, not a job that is red by
construction
paths:
- tests/local_testing/test_cache_preset_key.py
- tests/local_testing/test_caching.py
- tests/local_testing/test_caching_handler.py
- tests/local_testing/test_disk_cache_unit_tests.py
- tests/local_testing/test_gcs_cache_unit_tests.py
- tests/local_testing/test_prompt_caching.py
- tests/local_testing/test_responses_stream_cache_keys.py
- tests/local_testing/test_unit_test_caching.py
- reason: >-
The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
from a pull request; it needs a live gateway and provider credentials no PR job holds
@ -48,16 +45,6 @@ test_paths:
choice it informed is settled
paths:
- tests/code_coverage_tests/test_aio_http_image_conversion.py
- reason: >-
The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
their bodies into the live file of the same name. This one cannot follow either route yet:
its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
counterpart while 25 assertions fail against today's code, so what survives that rewrite
is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
behaviours still hold
paths:
- tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
- reason: >-
No job invokes this suite and its files mix pure transformation tests with ones driving live
vendor vector stores, so assigning them needs a per-file decision
@ -102,17 +89,6 @@ test_paths:
- tests/integration/sandbox/test_e2b_sandbox.py
- tests/integration/test_oci_integration.py
- tests/integration/test_oci_proxy_integration.py
- reason: >-
A unit test for the proxy-extras package that no job invokes, while the package's other
tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
this file requires. It also matches those keywords inside SQL comments, so two further
migrations are reported that are in fact fine. Wiring it up means deciding what to do about
the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
changed one breaks migrate deploy for existing installs
paths:
- tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
dockerfiles:
- reason: >-

23
.github/pull_request_template.md vendored
View file

@ -1,7 +1,10 @@
<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
everyday engineering language, extremely parsable and readable at a glance. This goes double for
the TLDR, User Flow, and Caveats sections -->
## TLDR
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
Problem this solves:
@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
## Caveats (if any)
<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
Call out known limitations, follow-up work, or anything a reviewer should watch out for
Include only the tiers that have caveats; drop the empty ones
- Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
operator must plan around it before rollout
- High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
unsafe to ship as is
- Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
- Low: anything else worth noting: naming, cleanup, an edge case nobody hits
Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
human reader
Leave this section empty if there are none -->
## QA runbook
@ -134,6 +149,6 @@ Example checklists:
- [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
-->
### Final Attestation
## Final Attestation
- [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

16
.github/scripts/assert_ci_coverage.py vendored
View file

@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
)
def _workflow_named_tokens() -> frozenset[str]:
"""Test tokens a GitHub Actions job names directly.
A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
workflow names it, so the slice check has to credit those the same way the census does.
"""
return _invoked_test_tokens(
scalar
for path in _config_files()
if path != CIRCLECI_CONFIG
for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
)
def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
slices: Final = _slices()
named_by_workflow: Final = _workflow_named_tokens()
globbed: Final = tuple(
path
for path in _test_files()
@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
)
for path in globbed
if not allowlist.covers_test(path)
and not any(_token_covers(token, path) for token in named_by_workflow)
and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
)

198
.github/scripts/e2e_egress_sentinel.py vendored Executable file
View file

@ -0,0 +1,198 @@
"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
`serve` pins each provider host (`--host`) to a local sink address in the hosts
file and binds a counting listener on that address, so any connection the proxy
or the record/replay edge opens to a real provider is redirected to the sink,
recorded as one line in `--hits-file`, and never leaves the box. The record and
replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
pinned provider names), so in a clean replay the sink sees nothing; a single hit
means a provider call escaped the bundle. `assert-empty` turns that hit file into
the pass/fail check.
Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
editing the hosts file both need root); `--sink-address`, `--port`, and
`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
a high port under test.
"""
# ruff: noqa: T201 # CLI script: its stdout/stderr progress and results are the interface
from __future__ import annotations
import argparse
import json
import os
import signal
import socket
import sys
import threading
import time
from dataclasses import dataclass
from pathlib import Path
from types import FrameType
from typing import Final
_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
_BLOCK_END: Final = "# END e2e-egress-sentinel"
@dataclass(frozen=True, slots=True)
class ServeConfig:
hosts: tuple[str, ...]
sink_address: str
ports: tuple[int, ...]
hits_file: Path
hosts_file: Path
ready_file: Path | None
pid_file: Path | None
def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
original = hosts_file.read_bytes() if hosts_file.exists() else b""
hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
return original
def _restore_pins(hosts_file: Path, original: bytes) -> None:
hosts_file.write_bytes(original)
def _bind(sink_address: str, port: int) -> socket.socket:
listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind((sink_address, port))
listener.listen(128)
return listener
@dataclass(frozen=True, slots=True)
class _HitLog:
path: Path
_lock: threading.Lock
def record(self, *, port: int, peer: tuple[str, int]) -> None:
entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
with self._lock:
with self.path.open("a", encoding="utf-8") as handle:
handle.write(entry + "\n")
def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
while not stop.is_set():
try:
conn, peer = listener.accept()
except OSError:
return
hits.record(port=port, peer=(peer[0], peer[1]))
try:
conn.close()
except OSError:
pass
def serve(config: ServeConfig) -> int:
config.hits_file.write_text("", encoding="utf-8")
original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
try:
listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
except OSError as exc:
_restore_pins(config.hosts_file, original_hosts)
print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
return 1
stop = threading.Event()
hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
threads = tuple(
threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
for listener, port in zip(listeners, config.ports)
)
for thread in threads:
thread.start()
def _handle(_signum: int, _frame: FrameType | None) -> None:
stop.set()
for listener in listeners:
try:
listener.close()
except OSError:
pass
signal.signal(signal.SIGTERM, _handle)
signal.signal(signal.SIGINT, _handle)
if config.pid_file is not None:
config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
if config.ready_file is not None:
config.ready_file.write_text("ready", encoding="utf-8")
print(
f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
f"on port(s) {', '.join(str(p) for p in config.ports)}",
flush=True,
)
stop.wait()
_restore_pins(config.hosts_file, original_hosts)
if config.ready_file is not None and config.ready_file.exists():
config.ready_file.unlink()
if config.pid_file is not None and config.pid_file.exists():
config.pid_file.unlink()
return 0
def assert_empty(hits_file: Path) -> int:
if not hits_file.exists():
print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
return 0
hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
if not hits:
print("egress sentinel recorded no provider calls: zero egress")
return 0
print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
for line in hits:
print(f" {line}", file=sys.stderr)
return 1
def _serve_from_args(args: argparse.Namespace) -> int:
config = ServeConfig(
hosts=tuple(args.host),
sink_address=args.sink_address,
ports=tuple(args.port),
hits_file=Path(args.hits_file),
hosts_file=Path(args.hosts_file),
ready_file=Path(args.ready_file) if args.ready_file else None,
pid_file=Path(args.pid_file) if args.pid_file else None,
)
return serve(config)
def main(argv: tuple[str, ...]) -> int:
parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
sub = parser.add_subparsers(dest="command", required=True)
serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
serve_parser.add_argument("--sink-address", default="127.0.0.1")
serve_parser.add_argument("--port", action="append", type=int, default=None)
serve_parser.add_argument("--hits-file", required=True)
serve_parser.add_argument("--hosts-file", default="/etc/hosts")
serve_parser.add_argument("--ready-file", default=None)
serve_parser.add_argument("--pid-file", default=None)
assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
assert_parser.add_argument("--hits-file", required=True)
args = parser.parse_args(argv)
if args.command == "serve":
if args.port is None:
args.port = [443]
return _serve_from_args(args)
return assert_empty(Path(args.hits_file))
if __name__ == "__main__":
raise SystemExit(main(tuple(sys.argv[1:])))

55
.github/scripts/e2e_fetch_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -euo pipefail
REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
BASE_BRANCH="${3:?base branch required}"
DEST_DIR="${4:?destination bundle dir required}"
: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
WORKDIR="$(mktemp -d)"
trap 'rm -rf "${WORKDIR}"' EXIT
echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
SELECTED="$(
gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
--jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
| jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
)"
if [[ -z "${SELECTED}" ]]; then
echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
exit 1
fi
RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
if [[ -z "${TARBALL}" ]]; then
echo "downloaded artifact contained no tarball" >&2
exit 1
fi
SIDECAR="${TARBALL}.sha256"
if [[ ! -f "${SIDECAR}" ]]; then
echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
exit 1
fi
echo "verifying bundle against its recorded sha256 digest"
( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
mkdir -p "${DEST_DIR}"
tar xzf "${TARBALL}" -C "${DEST_DIR}"
echo "extracted bundle into ${DEST_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

36
.github/scripts/e2e_pack_fixture_bundle.sh vendored Executable file
View file

@ -0,0 +1,36 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -ne 2 ]]; then
echo "usage: $0 <bundle-dir> <out-tarball>" >&2
exit 2
fi
BUNDLE_DIR="$1"
OUT_TARBALL="$2"
MANIFEST="${BUNDLE_DIR}/manifest.json"
if [[ ! -f "${MANIFEST}" ]]; then
echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
exit 1
fi
echo "packing fixture bundle from ${BUNDLE_DIR}"
python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print(' format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
if [[ "${TEST_DIRS}" -eq 0 ]]; then
echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
exit 1
fi
echo " ${TEST_DIRS} recorded test director(ies)"
mkdir -p "$(dirname "${OUT_TARBALL}")"
tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
OUT_BASE="$(basename "${OUT_TARBALL}")"
( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
cat "${OUT_DIR}/${OUT_BASE}.sha256"

13
.github/workflows/_test-unit-base.yml vendored
View file

@ -27,7 +27,7 @@ on:
default: 20
job-timeout-minutes:
description: >-
Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
the per-step ceilings on the setup steps below, and 5 for the runner
overhead the job clock charges but no step owns (job init, step
transitions, post-job cleanup). That headroom is what makes the test
@ -36,7 +36,7 @@ on:
arithmetic, so the sum is passed in rather than computed.
required: false
type: number
default: 55
default: 60
max-failures:
description: "Stop after this many failures"
required: false
@ -103,6 +103,11 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 8
@ -144,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \
--reruns-delay 1 \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
else
@ -156,7 +161,7 @@ jobs:
--reruns-delay 1 \
--dist="${DIST}" \
--durations=20 \
--cov=./litellm \
--cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \
--cov-config=pyproject.toml
fi

4
.github/workflows/check-ui-api-types.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.relevant == 'true'
uses: ./.github/actions/cache-cargo-build
- name: Install backend dependencies
if: steps.changes.outputs.relevant == 'true'
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

237
.github/workflows/e2e_record_replay.yml vendored Normal file
View file

@ -0,0 +1,237 @@
name: "E2E Record and Replay"
on:
schedule:
- cron: "0 8 * * 6"
- cron: "0 8 * * 1-5"
workflow_dispatch:
inputs:
mode:
description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
type: choice
options:
- record
- replay
default: record
permissions:
contents: read
jobs:
record:
name: "Record the e2e suite against real providers"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 6' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Record the replayable e2e lane
env:
E2E_FIXTURE_MODE: record
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Pack the fixture bundle
run: |
.github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
- name: Publish the fixture bundle
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: e2e-fixtures-bundle
path: |
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
if-no-files-found: error
retention-days: 30
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log
replay:
name: "Replay the e2e suite from the pinned bundle with zero egress"
if: >-
(github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
(github.event.schedule == '0 8 * * 1-5' ||
(github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
runs-on: ubuntu-latest
timeout-minutes: 45
permissions:
contents: read
actions: read
services:
postgres:
image: postgres:16.6
env:
POSTGRES_USER: llmproxy
POSTGRES_PASSWORD: dbpassword9090
POSTGRES_DB: litellm
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U llmproxy"
--health-interval 5s
--health-timeout 5s
--health-retries 10
env:
DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
LITELLM_MASTER_KEY: sk-e2e-record-replay
LITELLM_LOCAL_MODEL_COST_MAP: "True"
GH_TOKEN: ${{ github.token }}
OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Fetch the pinned fixture bundle by digest
env:
BASE_BRANCH: ${{ github.ref_name }}
run: |
.github/scripts/e2e_fetch_fixture_bundle.sh \
"${GITHUB_REPOSITORY}" \
e2e-fixtures-bundle \
"${BASE_BRANCH}" \
tests/e2e/.fixtures
- name: Start the proxy
run: |
nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
for _ in $(seq 1 90); do
if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
exit 0
fi
sleep 2
done
echo "proxy never became live"
tail -n 100 proxy.log
exit 1
- name: Start the egress sentinel
run: |
# shellcheck disable=SC2024 # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
--host api.openai.com \
--host api.anthropic.com \
--hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
--ready-file "${RUNNER_TEMP}/egress-ready" \
--pid-file "${RUNNER_TEMP}/egress.pid" \
> "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
for _ in $(seq 1 30); do
if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 0
fi
sleep 1
done
echo "egress sentinel never became ready"
cat "${RUNNER_TEMP}/egress-sentinel.log"
exit 1
- name: Replay the replayable e2e lane
env:
E2E_FIXTURE_MODE: replay
run: |
uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
- name: Stop the egress sentinel and assert zero provider egress
if: always()
run: |
if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
sleep 2
fi
python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
- name: Show proxy log on failure
if: failure()
run: tail -n 300 proxy.log

33
.github/workflows/image-scan.yml vendored
View file

@ -17,10 +17,14 @@ on:
- backend/Dockerfile
- backend/main.py
- docker/component_entrypoint.sh
- docker/entrypoint.sh
- litellm/proxy/prisma_migration.py
- litellm-proxy-extras/**
- tests/proxy_migration_tests/**
- uv.lock
- ui/litellm-dashboard/package-lock.json
- ui/Dockerfile
- ui/nginx.conf
- .github/workflows/image-scan.yml
schedule:
- cron: "41 6 * * *"
@ -183,6 +187,35 @@ jobs:
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
ui-image:
name: ui-image
runs-on: ubuntu-latest
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build UI image
run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
env:
LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
run: |
python -m pip install "pytest==9.0.3"
python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
backend-image:
name: backend-image
runs-on: ubuntu-latest

3
.github/workflows/mutation-test.yml vendored
View file

@ -53,6 +53,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml

3
.github/workflows/publish-basedpyright-base-counts.yml vendored
View file

@ -43,6 +43,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Cache Prisma binaries
uses: ./.github/actions/cache-prisma-binaries

6
.github/workflows/test-code-quality.yml vendored
View file

@ -56,6 +56,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
@ -128,6 +131,9 @@ jobs:
- name: check_e2e_no_raw_requests
run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
- name: check_migrations_no_data_rewrites
run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
- name: memory_test
run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py

15
.github/workflows/test-linting.yml vendored
View file

@ -67,6 +67,17 @@ jobs:
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-lint-
- name: Clean Python cache
if: steps.changes.outputs.decision != 'skip'
run: |
@ -78,6 +89,10 @@ jobs:
run: |
uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

6
.github/workflows/test-mcp.yml vendored
View file

@ -47,6 +47,10 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
@ -56,4 +60,4 @@ jobs:
- name: Run MCP tests
if: steps.changes.outputs.decision != 'skip'
run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

145
.github/workflows/test-postgres.yml vendored Normal file
View file

@ -0,0 +1,145 @@
name: "Postgres Tests"
on:
pull_request:
branches:
- main
- litellm_internal_staging
- litellm_oss_staging
- "litellm_**"
push:
branches:
- main
- litellm_internal_staging
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
postgres:
name: ${{ matrix.shard }}
runs-on: ubuntu-latest
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
services:
postgres:
image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: litellm_test
ports:
- 5432:5432
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 10
strategy:
fail-fast: false
matrix:
include:
- shard: proxy-behavior
test-path: "tests/proxy_behavior"
seed: db-push
workers: 0
timeout-minutes: 25
job-timeout-minutes: 50
- shard: proxy-security
test-path: "tests/proxy_security_tests"
seed: db-push
workers: 0
timeout-minutes: 15
job-timeout-minutes: 40
- shard: schema-migration
test-path: "tests/proxy_migration_tests"
seed: none
workers: 0
timeout-minutes: 20
job-timeout-minutes: 45
env:
DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
timeout-minutes: 3
with:
persist-credentials: false
- name: Detect relevant changes
id: changes
timeout-minutes: 2
uses: ./.github/actions/detect-changes
- name: Set up Python
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Cache uv dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-postgres-
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 12
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 3
uses: ./.github/actions/cache-prisma-binaries
- name: Generate Prisma client
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: 5
run: |
uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
- name: Seed database schema
if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
timeout-minutes: 10
run: |
uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
- name: Run tests
if: steps.changes.outputs.decision != 'skip'
timeout-minutes: ${{ matrix.timeout-minutes }}
env:
TEST_PATH: ${{ matrix.test-path }}
WORKERS: ${{ matrix.workers }}
run: |
if [ "${WORKERS}" = "0" ]; then
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
else
uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
fi

3
.github/workflows/test-terraform-provider.yml vendored
View file

@ -88,6 +88,9 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

4
.github/workflows/test-unit-documentation.yml vendored
View file

@ -67,6 +67,10 @@ jobs:
restore-keys: |
${{ runner.os }}-uv-
- name: Cache the Rust build
if: steps.changes.outputs.decision != 'skip'
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |

54
.github/workflows/test-unit.yml vendored
View file

@ -55,7 +55,7 @@ jobs:
workers: 2
reruns: 1
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: enterprise-routing
artifact-name: enterprise-routing
@ -67,7 +67,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: integrations
artifact-name: integrations
@ -75,7 +75,7 @@ jobs:
workers: 2
reruns: 3
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: Vertex AI
artifact-name: llm-vertex-ai
@ -83,7 +83,7 @@ jobs:
workers: 1
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: All Other Providers
artifact-name: llm-other-providers
@ -91,7 +91,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: misc
artifact-name: misc
@ -122,7 +122,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-auth
artifact-name: proxy-auth
@ -134,7 +134,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-endpoints
artifact-name: proxy-endpoints
@ -164,14 +164,15 @@ jobs:
tests/test_litellm/proxy/public_endpoints
tests/test_litellm/proxy/prompts
tests/test_litellm/proxy/rag_endpoints
tests/test_litellm/proxy/rerank_endpoints
tests/test_litellm/proxy/realtime_endpoints
tests/test_litellm/proxy/ui_crud_endpoints
tests/test_litellm/proxy/config_resolvers
tests/test_litellm/proxy/utils
workers: 2
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-server
artifact-name: proxy-server
@ -179,7 +180,7 @@ jobs:
workers: 4
reruns: 2
timeout-minutes: 60
job-timeout-minutes: 95
job-timeout-minutes: 100
- shard: proxy-infra
artifact-name: proxy-infra
@ -195,10 +196,39 @@ jobs:
tests/test_litellm/proxy/types_utils
tests/test_litellm/proxy/logging_endpoints
tests/test_litellm/proxy/test_*.py
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: caching-local
artifact-name: caching-local
test-path: >-
tests/local_testing/test_cache_preset_key.py
tests/local_testing/test_caching_handler.py
tests/local_testing/test_prompt_caching.py
tests/local_testing/test_responses_stream_cache_keys.py
tests/local_testing/test_unit_test_caching.py
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
- shard: proxy-extras
artifact-name: proxy-extras
test-path: "tests/litellm-proxy-extras"
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: enterprise-package
artifact-name: enterprise-package
test-path: "tests/enterprise"
workers: 4
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
- shard: responses-caching-types
artifact-name: responses-caching-types
@ -209,7 +239,7 @@ jobs:
workers: 2
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 55
job-timeout-minutes: 60
uses: ./.github/workflows/_test-unit-base.yml
with:
test-path: ${{ matrix.test-path }}

3
.github/workflows/weekly_load_anomaly.yml vendored
View file

@ -47,6 +47,9 @@ jobs:
with:
version: "0.10.9"
- name: Cache the Rust build
uses: ./.github/actions/cache-cargo-build
- name: Install dependencies
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

7
CLAUDE.md
View file

@ -37,13 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis
- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
- don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
@ -79,6 +80,8 @@ Do not put names of customers or customer company names in code, PR descriptions
CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
- Composition over inheritance

4
Dockerfile
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
backend/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

32
basedpyright-code-budget.json
View file

@ -1,18 +1,18 @@
{
"reportAny": {
"limit": 19955
"limit": 18505
},
"reportArgumentType": {
"limit": 2566
"limit": 2564
},
"reportAssignmentType": {
"limit": 320
},
"reportAttributeAccessIssue": {
"limit": 488
"limit": 483
},
"reportCallIssue": {
"limit": 114
"limit": 113
},
"reportConstantRedefinition": {
"limit": 40
@ -24,7 +24,7 @@
"limit": 19
},
"reportExplicitAny": {
"limit": 6049
"limit": 5976
},
"reportFunctionMemberAccess": {
"limit": 7
@ -45,7 +45,7 @@
"limit": 35
},
"reportInvalidTypeForm": {
"limit": 35
"limit": 34
},
"reportInvalidTypeVarUse": {
"limit": 2
@ -54,10 +54,10 @@
"limit": 0
},
"reportMissingParameterType": {
"limit": 5663
"limit": 5659
},
"reportMissingTypeArgument": {
"limit": 15555
"limit": 15504
},
"reportMissingTypeStubs": {
"limit": 40
@ -72,7 +72,7 @@
"limit": 0
},
"reportOptionalMemberAccess": {
"limit": 1061
"limit": 1058
},
"reportOptionalOperand": {
"limit": 0
@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
"limit": 1823
"limit": 1810
},
"reportRedeclaration": {
"limit": 8
@ -99,31 +99,31 @@
"limit": 0
},
"reportUnknownArgumentType": {
"limit": 44655
"limit": 44530
},
"reportUnknownLambdaType": {
"limit": 109
},
"reportUnknownMemberType": {
"limit": 39011
"limit": 38828
},
"reportUnknownParameterType": {
"limit": 19885
"limit": 19847
},
"reportUnknownVariableType": {
"limit": 30569
"limit": 30386
},
"reportUnnecessaryCast": {
"limit": 117
},
"reportUnnecessaryComparison": {
"limit": 699
"limit": 697
},
"reportUnnecessaryContains": {
"limit": 5
},
"reportUnnecessaryIsInstance": {
"limit": 836
"limit": 833
},
"reportUntypedBaseClass": {
"limit": 0

1
ci_cd/generate_model_prices_schema.py
View file

@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
"uses_embed_content",
"use_openai_responses_path",
"bedrock_converse_supports_strict_tools",
"thinking_always_on",
}
)

4
docker/Dockerfile.database
View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

4
docker/Dockerfile.non_root
View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

1
enterprise/enterprise_hooks/banned_keywords.py
View file

@ -21,6 +21,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BannedKeywords(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self):
banned_keywords_list = litellm.banned_keywords_list

1
enterprise/enterprise_hooks/blocked_user_list.py
View file

@ -18,6 +18,7 @@ from fastapi import HTTPException
class _ENTERPRISE_BlockedUserList(CustomLogger):
enforces_request_content: bool = True
# Class variables or attributes
def __init__(self, prisma_client: Optional[PrismaClient]):
self.prisma_client = prisma_client

14
enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
View file

@ -14,6 +14,8 @@ from litellm.constants import (
)
if TYPE_CHECKING:
from prisma import models as prisma_models
from litellm.integrations.prometheus import PrometheusLogger
from litellm.proxy._types import LiteLLM_ManagedObjectTable
from litellm.proxy.utils import PrismaClient, ProxyLogging
@ -351,7 +353,7 @@ class CheckBatchCost:
return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
async def _finalize_unbilled_terminal_job(
self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
) -> None:
"""Persist a terminal batch that has nothing billable, converting any raw
provider file ids to managed ids, and take it out of the poll page."""
@ -966,6 +968,16 @@ class CheckBatchCost:
)
elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
from litellm.proxy.openai_files_endpoints.common_utils import (
_completed_batch_safe_to_retire,
)
if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
verbose_proxy_logger.info(
f"CheckBatchCost: batch {batch_id} is completed but its output file id "
f"has not appeared yet; leaving job {job.id} for the next poll cycle"
)
continue
await self._finalize_unbilled_terminal_job(job, response)
# Record polling run metrics (always, even if nothing was processed)

87
enterprise/litellm_enterprise/proxy/hooks/managed_files.py
View file

@ -45,6 +45,8 @@ from litellm.proxy._types import (
UserAPIKeyAuth,
)
from litellm.proxy.openai_files_endpoints.common_utils import (
FILE_LIST_CONTINUATION_CHUNK_SIZE,
MAX_FILE_LIST_LIMIT,
_is_base64_encoded_unified_file_id,
apply_unified_file_ids,
ensure_batch_response_managed_file_ids,
@ -54,6 +56,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
map_raw_file_ids_to_unified,
normalize_mime_type_for_provider,
resolve_managed_output_file_model_name,
validate_file_list_limit,
validate_file_list_purpose,
)
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
request_tags_from_metadata,
@ -63,9 +67,9 @@ from litellm.types.llms.openai import ( # pyright: ignore[reportAttributeAccess
AsyncCursorPage,
ChatCompletionFileObject,
CreateFileRequest,
FileListPage,
FileObject,
OpenAIFileObject,
OpenAIFilesPurpose,
ResponsesAPIResponse,
)
from litellm.types.utils import (
@ -144,7 +148,14 @@ class _ManagedFileRow(Protocol):
class _ManagedFileTableActions(Protocol):
async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
async def find_many(
self,
where: Mapping[str, object],
take: int = ...,
order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
cursor: Mapping[str, str] = ...,
skip: int = ...,
) -> Sequence[_ManagedFileRow]: ...
async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
@ -1365,12 +1376,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
async def afile_list(
self,
purpose: Optional[OpenAIFilesPurpose],
purpose: Optional[str],
litellm_parent_otel_span: Optional[Span],
user_api_key_dict: UserAPIKeyAuth,
limit: Optional[int] = None,
after: Optional[str] = None,
**data: Dict,
) -> List[OpenAIFileObject]:
"""Handled in files_endpoints.py"""
return []
) -> FileListPage:
"""List the managed files the caller owns, newest first.
Pagination is keyset based on ``unified_file_id`` so a key that owns
every file on the proxy still reads one bounded page at a time.
``purpose`` is applied after parsing, because the managed file table
keeps it inside the ``file_object`` blob instead of a column, and rows
whose blob will not parse drop out there too, so a chunk of rows can
yield fewer matches than the page holds. Successive chunks are read
until the page is full or the caller's rows run out, which keeps
``data`` non-empty while matches remain and its last id usable as the
next cursor. A first chunk that fills the page costs one query; once a
scan has to continue past it, the chunk widens to
``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
that many rows instead of one per page. That bound is per query, not
per request: the work is still linear in the rows the caller owns, and
a filter matching nothing reads every one of them, with no index
covering either the owner filter or the sort.
"""
validate_file_list_limit(limit)
validate_file_list_purpose(purpose)
owner_filter: Final = build_owner_filter(user_api_key_dict)
if owner_filter is None:
return FileListPage(**build_list_page([]))
if after:
cursor_row = await _managed_file_table(self.prisma_client).find_first(
where={**owner_filter, "unified_file_id": after}
)
if cursor_row is None:
raise ProxyException(
message=f"Invalid 'after' cursor: no file found with id '{after}'.",
type="invalid_request_error",
param="after",
code=400,
openai_code="invalid_value",
)
page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
matches: Final[List[OpenAIFileObject]] = []
cursor_id = after
chunk_size = page_size + 1
while len(matches) <= page_size:
cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
chunk = await _managed_file_table(self.prisma_client).find_many(
where=owner_filter,
take=chunk_size,
order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
**cursor_args,
)
matches.extend(
parsed_file_object.model_copy(update={"id": row.unified_file_id})
for row in chunk
if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
and (purpose is None or parsed_file_object.purpose == purpose)
)
if len(chunk) < chunk_size:
break
cursor_id = chunk[-1].unified_file_id
chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
def _is_batch_polling_enabled(self) -> bool:
"""

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.58"
version = "0.1.60"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.58"
version = "0.1.60"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
gateway/Dockerfile
View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

8
helm/litellm/values.yaml
View file

@ -428,9 +428,11 @@ ui:
maxUnavailable: ""
podAnnotations: {}
# Same shape as the gateway blocks of the same name. The nginx runtime
# writes its pid, cache, and proxy temp files under the image's root
# filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
# emptyDir volumes mounted over those paths.
# writes its pid, cache, and proxy temp files under /tmp, so it boots as
# any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
# true` here needs an emptyDir volume mounted over /tmp. Images before
# the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
# run as a non-root uid at all.
podLabels: {}
podSecurityContext: {}
securityContext: {}

4
litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql
View file

@ -1,4 +0,0 @@
UPDATE "LiteLLM_SpendLogs"
SET "created_at" = "endTime",
"updated_at" = "endTime"
WHERE "created_at" > "endTime" + interval '1 hour';

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.88"
version = "0.4.89"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.88"
version = "0.4.89"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

12
litellm/__init__.py
View file

@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
)
log_raw_request_response: bool = False
log_client_error_tracebacks: bool = False
request_correlation_in_logs: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
@ -463,6 +464,11 @@ prometheus_metrics_config: Optional[List] = None
prometheus_exclude_metrics: Optional[List[str]] = None
prometheus_exclude_labels: Optional[List[str]] = None
prometheus_emit_stream_label: bool = False
prometheus_deployment_and_latency_caller_identity: Literal[
"api_key_alias",
"user_email",
"both",
] = "api_key_alias"
# Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
# `litellm_proxy_failed_requests_metric`. Off by default to preserve the
# pre-unification label set so existing dashboards / recording rules keyed on
@ -1628,6 +1634,9 @@ if TYPE_CHECKING:
AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
)
from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
from .llms.together_ai.chat.transformation import (
TogetherAIChatConfig as TogetherAIChatConfig,
)
from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
VertexGeminiConfig as VertexGeminiConfig,
@ -1801,6 +1810,9 @@ if TYPE_CHECKING:
from .llms.gemini.interactions.transformation import (
GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
)
from .llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig as VertexAIInteractionsConfig,
)
from .llms.openai.chat.o_series_transformation import (
OpenAIOSeriesConfig as OpenAIOSeriesConfig,
OpenAIOSeriesConfig as OpenAIO1Config,

10
litellm/_lazy_imports_registry.py
View file

@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
"AmazonAnthropicClaudeMessagesConfig",
"AmazonMantleMessagesConfig",
"TogetherAIConfig",
"TogetherAIChatConfig",
"NLPCloudConfig",
"VertexGeminiConfig",
"GoogleAIStudioGeminiConfig",
@ -242,6 +243,7 @@ LLM_CONFIG_NAMES: Final = (
"OpenRouterResponsesAPIConfig",
"BedrockMantleResponsesAPIConfig",
"GoogleAIStudioInteractionsConfig",
"VertexAIInteractionsConfig",
"OpenAIOSeriesConfig",
"AnthropicSkillsConfig",
"BaseSkillsAPIConfig",
@ -740,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
"AmazonMantleMessagesConfig",
),
"TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
"TogetherAIChatConfig": (
".llms.together_ai.chat.transformation",
"TogetherAIChatConfig",
),
"NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
"VertexGeminiConfig": (
".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
@ -977,6 +983,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
".llms.gemini.interactions.transformation",
"GoogleAIStudioInteractionsConfig",
),
"VertexAIInteractionsConfig": (
".llms.vertex_ai.interactions.transformation",
"VertexAIInteractionsConfig",
),
"OpenAIOSeriesConfig": (
".llms.openai.chat.o_series_transformation",
"OpenAIOSeriesConfig",

24
litellm/_logging.py
View file

@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str:
return _redact_string(value)
def _substituted_color_message(record: logging.LogRecord) -> str | None:
"""Render a record's ``color_message`` against its args, or None if absent.
uvicorn's colorized formatter re-renders `color_message` against
record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead
of using the already-formatted record.msg, so it has to be substituted
before args are cleared or it is later formatted with no args and prints
the raw "%s://%s:%d" placeholders instead of the URL.
"""
color_message: Final = record.__dict__.get("color_message")
if not isinstance(color_message, str) or not record.args:
return None
try:
return color_message % record.args
except TypeError:
return color_message
class SecretRedactionFilter(logging.Filter):
"""Scrubs known secret/credential patterns from log records."""
@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter):
if not _ENABLE_SECRET_REDACTION:
return True
# Runs before args are cleared, and before the extra-field loop below
# that redacts the substituted result.
substituted_color_message: Final = _substituted_color_message(record)
if substituted_color_message is not None:
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
try:
record.msg = _redact_string(record.getMessage())
record.args = None

10
litellm/_redis.py
View file

@ -665,8 +665,16 @@ def get_redis_async_client(
cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
cluster_kwargs.setdefault("socket_keepalive", True)
# A single node's client-side timeout must reset only that node's connections,
# not tear down the whole cluster client for every concurrent caller.
from litellm.caching.redis_cluster_node_isolation import (
get_litellm_async_redis_cluster_class,
)
async_redis_cluster_class: Final = get_litellm_async_redis_cluster_class()
# Create async RedisCluster with IAM token as password if available
cluster_client: Final = async_redis.RedisCluster(
cluster_client: Final = async_redis_cluster_class(
startup_nodes=new_startup_nodes,
**cluster_kwargs,
)

38
litellm/a2a_protocol/card_resolver.py
View file

@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
from litellm._logging import verbose_logger
@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
{
"jsonrpc": "JSONRPC",
"http+json": "HTTP+JSON",
"grpc": "GRPC",
}
)
_LEGACY_PROTOCOL_VERSION: Final = "0.3"
def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
"""
Canonicalize the supported interfaces of spec-adjacent agent cards.
Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
case-sensitively against its uppercase TransportProtocol constants and fails
with "no compatible transports found." for spec-adjacent casings.
The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
proto parsing rejects. A mis-cased binding fingerprints such a server, so its
declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
its v0.3 compat transport, which speaks that dialect.
"""
normalized: Final = type(agent_card)()
normalized.CopyFrom(agent_card)
for interface in normalized.supported_interfaces:
canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
if canonical is None or canonical == interface.protocol_binding:
continue
interface.protocol_binding = canonical
interface.protocol_version = _LEGACY_PROTOCOL_VERSION
return normalized
def get_agent_card_url(agent_card: "AgentCard") -> str | None:
"""Return the agent endpoint URL from the resolved SDK card."""
url: Final = getattr(agent_card, "url", None)

13
litellm/a2a_protocol/main.py
View file

@ -73,6 +73,7 @@ except ImportError:
from litellm.a2a_protocol.card_resolver import (
LiteLLMA2ACardResolver,
get_agent_card_url,
normalize_agent_card_interfaces,
)
from litellm.a2a_protocol.exception_mapping_utils import (
handle_a2a_localhost_retry,
@ -782,13 +783,17 @@ async def create_a2a_client(
if extra_headers:
verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
agent_card: Final = normalize_agent_card_interfaces(
await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
)
a2a_client: Final = await create_client( # pyright: ignore[reportOptionalCall]
base_url,
agent_card,
client_config=ClientConfig( # pyright: ignore[reportOptionalCall]
httpx_client=httpx_client,
streaming=streaming,
),
resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
)
# Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
# the configured httpx client and this agent's headers without excavating
@ -799,9 +804,7 @@ async def create_a2a_client(
if extra_headers
else None
)
agent_card: Final = getattr(a2a_client, "_card", None)
if agent_card is not None:
a2a_client._litellm_agent_card = agent_card
a2a_client._litellm_agent_card = agent_card
verbose_logger.info("A2A client created for %s", base_url)

40
litellm/batches/batch_utils.py
View file

@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage(
)
def _provider_output_file_id(output_file_id: str) -> str:
"""
Resolve the file id the provider actually knows: unified ids yield their embedded
llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through.
"""
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
get_original_file_id,
)
unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id)
if not unified_file_id:
return get_original_file_id(output_file_id)
try:
extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s",
output_file_id,
e,
)
return output_file_id
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted)
return extracted
async def _fetch_batch_output_file_content(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content(
Required for Azure and other providers that need authentication
"""
from litellm.files.main import afile_content
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
)
if batch.output_file_id is None:
raise ValueError("Output file id is None cannot retrieve file content")
file_id = batch.output_file_id
is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id)
if is_base64_unified_file_id:
try:
file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id)
except (IndexError, AttributeError) as e:
verbose_logger.error(
"Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e
)
file_id: Final = _provider_output_file_id(batch.output_file_id)
# Build kwargs for afile_content with credentials from litellm_params
file_content_kwargs: Final = {

8
litellm/caching/_embedding_router.py
View file

@ -16,6 +16,7 @@ from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
if TYPE_CHECKING:
from litellm.router import Router
@ -60,6 +61,13 @@ def resolve_embedding_max_input_tokens(
return deployment_max_input_tokens
def resolve_embedding_timeout(configured_timeout: float | None) -> float:
"""Explicit cache setting first, else the short semantic-cache default."""
if configured_timeout is not None:
return configured_timeout
return SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
def truncate_embedding_input(prompt: str, embedding_model: str, max_input_tokens: int | None) -> str:
"""Keep only the first ``max_input_tokens`` tokens of ``prompt`` for the embedding call."""
if max_input_tokens is None:

5
litellm/caching/caching.py
View file

@ -98,6 +98,7 @@ class Cache:
qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002",
qdrant_semantic_cache_vector_size: int | None = None,
semantic_cache_embedding_max_input_tokens: int | None = None,
semantic_cache_embedding_timeout: float | None = None,
# GCP IAM authentication parameters
gcp_service_account: str | None = None,
gcp_ssl_ca_certs: str | None = None,
@ -124,6 +125,7 @@ class Cache:
qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
# Disk Cache Args
disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@ -195,6 +197,7 @@ class Cache:
embedding_model=redis_semantic_cache_embedding_model,
index_name=redis_semantic_cache_index_name,
embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
embedding_timeout=semantic_cache_embedding_timeout,
**kwargs,
)
elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
@ -211,6 +214,7 @@ class Cache:
index_name=valkey_semantic_cache_index_name,
startup_nodes=redis_startup_nodes,
embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
embedding_timeout=semantic_cache_embedding_timeout,
**kwargs,
)
elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
@ -223,6 +227,7 @@ class Cache:
embedding_model=qdrant_semantic_cache_embedding_model,
vector_size=qdrant_semantic_cache_vector_size,
embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
embedding_timeout=semantic_cache_embedding_timeout,
)
elif type == LiteLLMCacheType.LOCAL:
self.cache = InMemoryCache()

34
litellm/caching/qdrant_semantic_cache.py
View file

@ -16,7 +16,11 @@ from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import print_verbose
from litellm.constants import QDRANT_SCALAR_QUANTILE, QDRANT_VECTOR_SIZE
from litellm.constants import (
QDRANT_SCALAR_QUANTILE,
QDRANT_VECTOR_SIZE,
SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS,
)
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -26,6 +30,7 @@ from ._embedding_router import (
build_router_embedding_metadata,
resolve_embedding_max_input_tokens,
resolve_embedding_router,
resolve_embedding_timeout,
truncate_embedding_input,
)
from .base_cache import BaseCache
@ -37,6 +42,7 @@ if TYPE_CHECKING:
class QdrantSemanticCache(BaseCache):
CACHE_KEY_FIELD_NAME = "litellm_cache_key"
embedding_max_input_tokens: int | None = None
embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
def __init__(
self,
@ -49,6 +55,7 @@ class QdrantSemanticCache(BaseCache):
host_type=None,
vector_size=None,
embedding_max_input_tokens: int | None = None,
embedding_timeout: float | None = None,
):
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
@ -68,6 +75,7 @@ class QdrantSemanticCache(BaseCache):
self.similarity_threshold = similarity_threshold
self.embedding_model = embedding_model
self.embedding_max_input_tokens = embedding_max_input_tokens
self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE
headers = {}
@ -222,11 +230,15 @@ class QdrantSemanticCache(BaseCache):
input=embedding_input,
cache={"no-store": True, "no-cache": True},
metadata=build_router_embedding_metadata(metadata),
timeout=self.embedding_timeout,
num_retries=0,
)
return litellm.embedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
timeout=self.embedding_timeout,
num_retries=0,
)
async def _get_async_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> EmbeddingResponse:
@ -238,19 +250,25 @@ class QdrantSemanticCache(BaseCache):
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
if router is not None:
return await router.aembedding(
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
metadata=build_router_embedding_metadata(metadata),
timeout=self.embedding_timeout,
num_retries=0,
)
if router is not None
else litellm.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
timeout=self.embedding_timeout,
num_retries=0,
)
return await litellm.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
)
return await asyncio.wait_for(embedding_call, self.embedding_timeout)
def set_cache(self, key, value, **kwargs):
print_verbose(f"qdrant semantic-cache set_cache, kwargs: {kwargs}")

201
litellm/caching/redis_cluster_node_isolation.py Normal file
View file

@ -0,0 +1,201 @@
"""Bounds the blast radius of a single node's transient connection error on the async
Redis Cluster client.
redis-py's ``RedisCluster._execute_command`` responds to a ``ConnectionError`` or
``TimeoutError`` on ANY one node by tearing down every node's connections and flipping
the client into "needs reinitialization", which forces every other concurrent caller
sharing this client through one reinit lock until the whole cluster topology is
re-walked. Under real proxy load, a client-side socket timeout on a single node is a
routine event (the event loop was too busy to read the response before ``socket_timeout``
elapsed) and does not mean the cluster's topology moved, so treating it as a full-cluster
event turns one slow node into a proxy-wide latency spike while Redis itself stays
healthy -- confirmed live: pausing one of three local cluster nodes made every concurrent
command against the other two, untouched nodes stall for the full pause duration too.
``get_litellm_async_redis_cluster_class`` returns a ``RedisCluster`` subclass that resets
only the node that actually failed (mirroring what a plain, non-cluster Redis client
already does when one of its pooled connections errors), leaving every other node's
connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
topology changed.
redis-py 8.x fixed this upstream with gentler machinery than this override's
``node.disconnect()`` (which also kills connections other coroutines are mid-operation
on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
killed connection): it marks in-use connections for reconnect only after their current
operation completes, disconnects only the idle pooled ones, and defers reinitialization
to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
recovery API, the factory returns the base ``RedisCluster`` unmodified.
"""
import asyncio
from typing import TYPE_CHECKING, Final, Protocol
from litellm._logging import verbose_logger
if TYPE_CHECKING:
from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
class _ClusterNodeAttrs(Protocol):
"""The subset of ``redis.asyncio.cluster.ClusterNode`` this override reads. redis-py
ships no resolvable stub for these members under the repo's current types-redis pin,
so a plain attribute access resolves every downstream use to ``Unknown`` under strict
mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
own logic fully typed without a banned ``typing.cast``."""
async def execute_command(
self,
*args: object,
**kwargs: object, # kwargs-ok: mirrors redis-py's own ClusterNode.execute_command signature, a raw command dispatch with no fixed keyword contract
) -> object: ...
async def disconnect(self) -> None: ...
class _NodesManagerAttrs(Protocol):
_moved_exception: object
def get_node_from_slot(
self, slot: int, read_from_replicas: bool, load_balancing_strategy: object
) -> _ClusterNodeAttrs: ...
class _ClusterAttrs(Protocol):
RedisClusterRequestTTL: int
reinitialize_counter: int
reinitialize_steps: int
read_from_replicas: bool
load_balancing_strategy: object
nodes_manager: _NodesManagerAttrs
def get_node(self, node_name: str) -> _ClusterNodeAttrs: ...
async def _determine_slot(self, *args: object) -> int: ...
async def aclose(self) -> None: ...
#: redis-py versions this override's copied ``_execute_command`` body has been verified
#: against. A version outside this set may have changed the method's structure in a way
#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
#: construction logs a loud warning rather than silently trusting an unverified copy.
_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
def get_litellm_async_redis_cluster_class(
cluster_node_class: type | None = None,
) -> type["_AsyncRedisClusterType"]:
"""Returns the base ``RedisCluster`` when the installed redis-py already recovers a
node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
subclass with the per-node isolation fix for older versions whose upstream branch
tears down the whole cluster client.
``cluster_node_class`` exists for dependency injection in tests; production callers
leave it unset and the installed ``ClusterNode`` is used.
Imported lazily because this module is reachable from a base ``import litellm`` while
redis is not a base dependency. Cheap to call repeatedly: the underlying redis
submodules are cached in ``sys.modules`` after the first import.
"""
import redis
from redis.asyncio.cluster import (
ClusterNode as _AsyncClusterNode, # pyright: ignore[reportUnknownVariableType] # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
)
from redis.asyncio.cluster import (
RedisCluster as _BaseAsyncRedisCluster, # pyright: ignore[reportUnknownVariableType] # same stale-stub gap as the import above
)
from redis.cluster import get_node_name
from redis.commands import READ_COMMANDS
from redis.exceptions import (
AskError,
BusyLoadingError,
ClusterDownError,
ClusterError,
MaxConnectionsError,
MovedError,
SlotNotCoveredError,
TryAgainError,
)
from redis.exceptions import ConnectionError as _RedisConnectionError
from redis.exceptions import TimeoutError as _RedisTimeoutError
node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
if hasattr(node_class, "update_active_connections_for_reconnect"):
verbose_logger.debug(
"redis-py %s recovers a node-level connection error per-connection upstream; "
"using the base RedisCluster without litellm's node-isolation override.",
redis.__version__,
)
return _BaseAsyncRedisCluster
if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
verbose_logger.warning(
"redis-py %s is not in the set this cluster-teardown-storm fix was verified "
"against (%s). The per-node-isolation override may not match the installed library's "
"real _execute_command behavior.",
redis.__version__,
sorted(_VERIFIED_REDIS_VERSIONS),
)
class LiteLLMAsyncRedisCluster(
_BaseAsyncRedisCluster # pyright: ignore[reportUntypedBaseClass] # same stale-stub gap as the import above; the base class itself is unresolvable, not this subclass's own code
):
async def _execute_command(
self,
target_node: _ClusterNodeAttrs,
*args: object,
**kwargs: object, # kwargs-ok: overrides redis-py's own **kwargs signature; the keyword contract is defined by the Redis command being dispatched, not by this method
) -> object:
cluster: _ClusterAttrs = self
node = target_node
asking = moved = False
redirect_addr: str | None = None
ttl = cluster.RedisClusterRequestTTL
while ttl > 0:
ttl -= 1
try:
if asking:
assert redirect_addr is not None
node = cluster.get_node(node_name=redirect_addr)
await node.execute_command("ASKING")
asking = False
elif moved:
slot = await cluster._determine_slot(*args) # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch, which makes this identical private call from the same subclass
node = cluster.nodes_manager.get_node_from_slot(
slot,
cluster.read_from_replicas and args[0] in READ_COMMANDS,
(cluster.load_balancing_strategy if args[0] in READ_COMMANDS else None),
)
moved = False
return await node.execute_command(*args, **kwargs)
except (BusyLoadingError, MaxConnectionsError):
raise
except (_RedisConnectionError, _RedisTimeoutError):
# Reset only the node that actually failed instead of the upstream
# default (`await self.aclose()`, a full-cluster teardown that forces
# every other concurrent caller through the shared reinit lock).
await node.disconnect()
raise
except (ClusterDownError, SlotNotCoveredError):
await cluster.aclose()
await asyncio.sleep(0.25)
raise
except MovedError as e:
cluster.reinitialize_counter += 1
if cluster.reinitialize_steps and cluster.reinitialize_counter % cluster.reinitialize_steps == 0:
await cluster.aclose()
cluster.reinitialize_counter = 0
else:
cluster.nodes_manager._moved_exception = e # pyright: ignore[reportPrivateUsage] # mirrors upstream's own un-overridden branch; redis-py exposes no public setter for this
moved = True
except AskError as e:
redirect_addr = get_node_name(host=e.host, port=e.port)
asking = True
except TryAgainError:
if ttl < cluster.RedisClusterRequestTTL / 2:
await asyncio.sleep(0.05)
raise ClusterError("TTL exhausted.")
return LiteLLMAsyncRedisCluster

43
litellm/caching/redis_semantic_cache.py
View file

@ -18,6 +18,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
import litellm
from litellm._logging import print_verbose, verbose_logger
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
)
@ -27,6 +28,7 @@ from ._embedding_router import (
build_router_embedding_metadata,
resolve_embedding_max_input_tokens,
resolve_embedding_router,
resolve_embedding_timeout,
truncate_embedding_input,
)
from .base_cache import BaseCache
@ -47,6 +49,7 @@ class RedisSemanticCache(BaseCache):
DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
embedding_max_input_tokens: int | None = None
embedding_timeout: float = SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
def __init__(
self,
@ -58,6 +61,7 @@ class RedisSemanticCache(BaseCache):
embedding_model: str = "text-embedding-ada-002",
index_name: str | None = None,
embedding_max_input_tokens: int | None = None,
embedding_timeout: float | None = None,
**kwargs: object,
):
"""
@ -74,6 +78,8 @@ class RedisSemanticCache(BaseCache):
index_name: Name for the Redis index
embedding_max_input_tokens: Truncate prompts to this many tokens before
embedding; defaults to the Router deployment's configured max_input_tokens
embedding_timeout: Seconds a cache lookup may spend embedding the prompt before it
gives up and lets the request continue to the LLM
ttl: Default time-to-live for cache entries in seconds
**kwargs: Additional arguments passed to the Redis client
@ -99,6 +105,7 @@ class RedisSemanticCache(BaseCache):
self.distance_threshold = 1 - similarity_threshold
self.embedding_model = embedding_model
self.embedding_max_input_tokens = embedding_max_input_tokens
self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
# Set up Redis connection
if redis_url is None:
@ -349,6 +356,8 @@ class RedisSemanticCache(BaseCache):
input=embedding_input,
cache={"no-store": True, "no-cache": True},
metadata=build_router_embedding_metadata(metadata),
timeout=self.embedding_timeout,
num_retries=0,
),
)
else:
@ -358,6 +367,8 @@ class RedisSemanticCache(BaseCache):
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
timeout=self.embedding_timeout,
num_retries=0,
),
)
return embedding_response["data"][0]["embedding"]
@ -512,20 +523,26 @@ class RedisSemanticCache(BaseCache):
router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
embedding_input: Final = self._embedding_input(prompt, router)
embedding_call: Final = (
router.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
metadata=build_router_embedding_metadata(metadata),
timeout=self.embedding_timeout,
num_retries=0,
)
if router is not None
else litellm.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
timeout=self.embedding_timeout,
num_retries=0,
)
)
try:
if router is not None:
embedding_response = await router.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
metadata=build_router_embedding_metadata(metadata),
)
else:
embedding_response = await litellm.aembedding(
model=self.embedding_model,
input=embedding_input,
cache={"no-store": True, "no-cache": True},
)
embedding_response: Final = await asyncio.wait_for(embedding_call, self.embedding_timeout)
return embedding_response["data"][0]["embedding"]
except Exception as e:
print_verbose(f"Error generating async embedding: {e}")

3
litellm/caching/valkey_semantic_cache.py
View file

@ -30,6 +30,7 @@ from litellm._logging import print_verbose
from litellm._uuid import uuid
from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
from ._embedding_router import resolve_embedding_timeout
from .redis_semantic_cache import RedisSemanticCache
@ -62,6 +63,7 @@ class ValkeySemanticCache(RedisSemanticCache):
sync_client: Redis | None = None,
async_client: AsyncRedis | None = None,
embedding_max_input_tokens: int | None = None,
embedding_timeout: float | None = None,
**kwargs: Any,
):
if similarity_threshold is None:
@ -80,6 +82,7 @@ class ValkeySemanticCache(RedisSemanticCache):
self.similarity_threshold = similarity_threshold
self.embedding_model = embedding_model
self.embedding_max_input_tokens = embedding_max_input_tokens
self.embedding_timeout = resolve_embedding_timeout(embedding_timeout)
self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
self.key_prefix = f"{self.index_name}:"
self._index_dim: int | None = None

59
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
import json
import os
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
from openai.types.responses.custom_tool_param import CustomToolParam
from openai.types.responses.response_input_param import (
@ -21,6 +21,9 @@ from pydantic import BaseModel
import litellm
from litellm import ModelResponse
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
responses_reasoning_item_from_thinking_blocks,
)
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.bridges.completion_transformation import (
CompletionTransformationBridge,
@ -32,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
)
from litellm.responses.utils import normalize_responses_api_stream_options
from litellm.types.llms.openai import (
REASONING_EFFORT,
ChatCompletionAnnotation,
ChatCompletionReasoningItem,
ChatCompletionToolCallChunk,
@ -85,6 +89,22 @@ def _get_reasoning_items(
return []
def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]: # mutable-ok: API message payload
"""Reasoning input items for an assistant message.
Stored reasoning items win because they carry an id the Responses API minted; thinking
blocks are the fallback for turns that arrived over another API surface.
"""
items: Final = _get_reasoning_items(msg)
stored: Final = [_reasoning_item_to_response_input(item) for item in items] # mutable-ok: API message payload
if stored:
return stored
raw_blocks: Final = msg.get("thinking_blocks") or ()
blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks) # cast-ok: untyped client json
from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
return [] if from_thinking is None else [dict(from_thinking)] # mutable-ok: API message payload
def _build_reasoning_item(
item_id: str,
encrypted_content: str | None,
@ -372,8 +392,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
)
)
elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
if content:
input_items.append(
{ # mutable-ok: API message payload
"type": "message",
"role": "assistant",
"content": self._convert_content_to_responses_format(content, "assistant"),
}
)
for tool_call in tool_calls:
function = tool_call.get("function")
custom = tool_call.get("custom")
@ -400,15 +427,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
raise ValueError(f"tool call not supported: {tool_call}")
elif content is not None:
if role == "assistant":
for r_item in _get_reasoning_items(msg):
input_items.append(_reasoning_item_to_response_input(r_item))
input_items.extend(_reasoning_input_items(msg))
input_items.append(
{
{ # mutable-ok: API message payload
"type": "message",
"role": role,
"content": self._convert_content_to_responses_format(content, cast(str, role)),
}
)
elif role == "assistant":
input_items.extend(_reasoning_input_items(msg))
return input_items, instructions
@ -1086,22 +1114,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
)
# If string is passed, map with optional summary based on flag/env var
if reasoning_effort == "none":
return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
elif reasoning_effort == "high":
return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
elif reasoning_effort == "xhigh":
return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
elif reasoning_effort == "medium":
if reasoning_effort in get_args(REASONING_EFFORT):
return (
Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
)
elif reasoning_effort == "low":
return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
elif reasoning_effort == "minimal":
return (
Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
Reasoning(effort=reasoning_effort, summary="detailed")
if auto_summary_enabled
else Reasoning(effort=reasoning_effort)
)
return None

30
litellm/constants.py
View file

@ -48,6 +48,9 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
# Data URIs exceeding this are replaced with a size placeholder.
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
@ -146,6 +149,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
"x-litellm-adaptive-router-model",
"x-litellm-applied-guardrails",
"x-litellm-guardrail-scan-id",
"x-litellm-cache-key",
]
# Gemini model-specific minimal thinking budget constants
@ -436,6 +440,9 @@ DEFAULT_REQUEST_TIMEOUT_SECONDS: Final[float] = 6000.0
# deadline and connect handshake (see ``http_handler`` cached handler paths).
COMPLETION_HTTP_FALLBACK_SECONDS: Final[float] = 600.0
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: Final[float] = 5.0
SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS: Final[float] = float(
os.getenv("SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS", "5.0")
)
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS))))
request_timeout_explicitly_set: bool = "REQUEST_TIMEOUT" in os.environ
DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = float(os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)) # 10 minutes
@ -457,6 +464,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
]
STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
### SPEND TRACKING ###
DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@ -746,6 +755,7 @@ openai_compatible_endpoints: Final[list] = [
"api.groq.com/openai/v1",
"https://integrate.api.nvidia.com/v1",
"api.deepseek.com/v1",
"api.together.ai/v1",
"api.together.xyz/v1",
"app.empower.dev/api/v1",
"https://api.friendli.ai/serverless/v1",
@ -780,6 +790,7 @@ openai_compatible_endpoints: Final[list] = [
"https://pinstripes.io/v1",
"https://api.meta.ai/v1",
"https://api.cognition.ai/v1",
"https://api.scx.ai/v1",
]
@ -848,6 +859,7 @@ openai_compatible_providers: Final[list] = [
"darkbloom",
"meta", # Meta Model API (Muse Spark) - JSON-configured provider
"cognition",
"scx-ai",
]
openai_text_completion_compatible_providers: Final[list] = [ # providers that support `/v1/completions`
"together_ai",
@ -1351,6 +1363,8 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
LITELLM_METADATA_FIELD: Final = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD: Final = "metadata"
RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@ -1531,9 +1545,12 @@ TOOL_SPEND_TOP_TOOLS: Final = 100
SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3")))
RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2"))
SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)) # 1 minute
PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
@ -1553,6 +1570,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
# installations with large numbers of stale managed objects).
_batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
)
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
)
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
)
_background_interaction_cost_polling_env: Final = os.getenv(
"BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
).lower()
BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10)) # in seconds, increased from 10
PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

25
litellm/cost_calculator.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
TranscriptionUsageObjectTransformation,
)
from litellm.litellm_core_utils.llm_cost_calc.utils import (
@ -150,6 +151,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
}
)
_SPEECH_CALL_TYPES: Final = frozenset(
{
CallTypes.speech.value,
@ -912,6 +914,8 @@ def _get_usage_object(
usage_obj,
)
)
elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
elif isinstance(usage_obj, dict):
return Usage(**usage_obj)
elif isinstance(usage_obj, BaseModel):
@ -1288,6 +1292,10 @@ def completion_cost(
)
if tr_usage is not None:
_usage = tr_usage.model_dump()
elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
_usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
_usage
).model_dump()
else:
_usage = _usage
@ -1372,23 +1380,36 @@ def completion_cost(
if custom_pricing and litellm_logging_obj is not None:
_litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
if _litellm_params is not None:
_metadata = _litellm_params.get("metadata", {}) or {}
_video_model_info = _metadata.get("model_info", None)
_video_model_info = next(
(
model_info
for _metadata_key in ("metadata", "litellm_metadata")
if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
is not None
),
None,
)
usage_obj = getattr(completion_response, "usage", None)
duration_seconds: float | None = None
video_resolution: str | None = None
provider_reported_cost: float | None = None
if completion_response is not None and usage_obj:
# Handle both dict and Pydantic Usage object
if isinstance(usage_obj, dict):
duration_seconds = usage_obj.get("duration_seconds", None)
_vr = usage_obj.get("video_resolution", None)
provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
else:
duration_seconds = getattr(usage_obj, "duration_seconds", None)
_vr = getattr(usage_obj, "video_resolution", None)
provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
if _vr is not None:
video_resolution = str(_vr).strip().lower()
if _video_model_info is None and provider_reported_cost is not None:
return float(provider_reported_cost)
if duration_seconds is not None:
# Calculate cost based on video duration using video-specific cost calculation
from litellm.llms.openai.cost_calculation import (

46
litellm/images/main.py
View file

@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
from litellm.exceptions import LiteLLMUnknownProvider
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
from litellm.litellm_core_utils.mock_functions import mock_image_generation
from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@ -422,24 +423,32 @@ def image_generation(
aimg_generation=aimg_generation,
)
elif custom_llm_provider == "azure_ai":
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
)
api_base = AzureFoundryModelInfo.get_api_base(api_base)
api_key = AzureFoundryModelInfo.get_api_key(api_key)
if extra_headers is not None:
optional_params["extra_headers"] = extra_headers
default_headers = {
caller_header_names = frozenset(name.lower() for name in headers)
caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
auth_headers = (
headers
if caller_set_auth
else get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params_dict,
api_key_header="api-key",
)
)
request_headers: Final = {
"Content-Type": "application/json",
**auth_headers,
**headers,
}
# Only add api-key header if api_key is not None
# Azure AD authentication will use Authorization header instead
if api_key is not None:
default_headers["api-key"] = api_key
for k, v in default_headers.items():
if k not in headers:
headers[k] = v
model_response = azure_chat_completions.image_generation(
model=model,
@ -455,7 +464,7 @@ def image_generation(
api_version=api_version,
aimg_generation=aimg_generation,
client=client,
headers=headers,
headers=request_headers,
litellm_params=litellm_params_dict,
)
elif (
@ -846,6 +855,18 @@ def image_edit(
additional_drop_params=kwargs.get("additional_drop_params"),
)
if (
custom_llm_provider == "openai"
or custom_llm_provider == "azure"
or custom_llm_provider in litellm.openai_compatible_providers
):
image_edit_request_params.update(
flatten_form_field_values(
non_default_params,
extra_body if isinstance(extra_body, dict) else None,
)
)
# Pre Call logging
litellm_logging_obj.update_from_kwargs(
kwargs=kwargs,
@ -995,6 +1016,9 @@ async def aimage_edit(
response_format=response_format,
size=size,
user=user,
extra_headers=extra_headers,
extra_query=extra_query,
extra_body=extra_body,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
**kwargs,

14
litellm/integrations/callback_configs.json
View file

@ -294,12 +294,18 @@
"id": "newrelic",
"displayName": "New Relic",
"logo": "newrelic.png",
"supports_key_team_logging": false,
"supports_key_team_logging": true,
"dynamic_params": {
"NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
"newrelic_api_key": {
"type": "password",
"ui_name": "New Relic Ingest License Key",
"description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
"required": false
},
"newrelic_region": {
"type": "text",
"ui_name": "Record AI Content (default: true)",
"description": "Whether to record AI message content. Set to false to disable.",
"ui_name": "New Relic Region (us or eu)",
"description": "Data center region for this team's account. Defaults to us.",
"required": false
}
},

4
litellm/integrations/compression_interception/handler.py
View file

@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
import time
import uuid
from typing import Any, Final, cast
from typing import Any, ClassVar, Final, cast
from litellm._logging import verbose_logger
from litellm.compression import compress
@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
4. Build typed rerun plan with tool_result blocks from the compressed cache.
"""
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
def __init__(
self,
enabled: bool = True,

72
litellm/integrations/custom_logger.py
View file

@ -2,8 +2,8 @@
# On success, logs events to Promptlayer
import re
import traceback
from collections.abc import AsyncGenerator
from typing import TYPE_CHECKING, Any, Final, Optional
from collections.abc import AsyncGenerator, Mapping
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from pydantic import BaseModel
@ -60,6 +60,26 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
# Class variables or attributes
server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
enforces_request_content: bool = False
"""
Whether this hook's ``async_pre_call_hook`` judges the request payload itself.
False for the accounting hooks, which count a request rather than read it: rate limits,
parallel slots, budgets, cache lookups. Those must run once per request and never once per
record of a batch upload, which would charge a caller once for every line of their file.
Set it to True on a hook that inspects or rejects content, so that scanning a payload which
is not itself a request, such as one record of a batch input file, still reaches it. A
``CustomGuardrail`` does not need it; guardrails are dispatched by their own branch.
Judging content is necessary but not sufficient. A hook that also rewrites the payload for
routing, as the managed-files and managed-vector-store hooks do, stays False: a per-record
rewrite would read as a redaction and ship embedded in the record. Only the leaf class is
consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
"""
def __init__(
self,
turn_off_message_logging: bool = False,
@ -273,6 +293,54 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
Allow modifying / reviewing the response just after it's received from the deployment.
"""
async def async_post_call_failure_deployment_hook(
self,
request_data: Mapping[str, Any],
exception: Exception,
call_type: CallTypes | None,
fallback_depth: int | None = None,
) -> None:
"""
Called once per failed deployment attempt - attempt 1, every retry, and
every fallback chain step - because the router re-invokes the wrapped
function on each attempt, re-entering this hook's call site fresh
every time.
This is a DEPLOYMENT-LEVEL signal, distinct from the REQUEST-LEVEL
``async_log_failure_event``, which fires once per logical client
request behind a dedup gate. ``request_data`` is mostly this
attempt's own kwargs, with one exception: it omits
``attempted_targets``, the router's own bookkeeping of which fallback
targets this request has already tried, since that one object *is*
shared by reference across every hop of the live fallback walk.
Pairs with ``async_pre_call_deployment_hook`` and
``async_post_call_success_deployment_hook`` to complete the
pre-call/success/failure lifecycle for a single deployment attempt.
``fallback_depth`` is best-effort: ``None`` on the first attempt and on
any call made without a ``Router`` (a bare SDK call has no fallback
chain to be at a depth in), ``1`` on the first fallback hop, ``2`` on
the second, and so on. It reflects ``Router``'s own internal fallback
bookkeeping (``kwargs["fallback_depth"]``), not a value this hook
computes or guarantees the shape of across versions. It tracks
fallback hops only, not retries within the same model group - a
retry-only failure (no fallback yet) also reports ``None``. If an
override predates this field it's simply never passed, rather than
raising - safe to leave off an override written before it existed.
``exception`` is a same-class snapshot, not the exact object about to
be re-raised to the real caller: read it freely, but setting an
attribute on it (e.g. ``status_code``) has no effect on what the
caller actually receives.
Default: no-op. Opt in by overriding. Keep overrides fast - this
runs on the request's exception path, so a slow implementation
delays error propagation to the caller. The reported failure
duration is captured before this hook runs, so a slow override
doesn't inflate that metric, but the caller still waits for it.
"""
async def async_post_call_streaming_deployment_hook(
self,
request_data: dict,

7
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
LangfuseLLMObsOTELAttributes,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
)
@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
)
elif item_type == "function_call":
arguments_str = getattr(item, "arguments", "{}")
arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
arguments_obj = (
safe_json_loads(arguments_str, default={})
if isinstance(arguments_str, str)
else arguments_str
)
langfuse_tool_call = {
"id": getattr(item, "id", ""),
"name": getattr(item, "name", ""),

22
litellm/integrations/langsmith.py
View file

@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
return outputs
def _ensure_required_ids(self, data: dict, run_id: str | None):
resolved_id: Final = run_id or str(uuid.uuid4())
if "id" not in data or data["id"] is None:
run_id = str(uuid.uuid4())
data["id"] = run_id
data["id"] = resolved_id
if "trace_id" not in data or data["trace_id"] is None:
if run_id is not None and isinstance(run_id, str):
data["trace_id"] = run_id
# LangSmith rejects the whole ingest batch unless a root run's trace_id
# equals the run id embedded in the first segment of dotted_order
posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
"dotted_order" not in data or data["dotted_order"] is None
)
if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
data["trace_id"] = resolved_id
if "dotted_order" not in data or data["dotted_order"] is None:
if run_id is not None and isinstance(run_id, str):
data["dotted_order"] = self.make_dot_order(run_id=run_id)
data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
def _prepare_log_data(
self,
@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
metadata = _litellm_params.get("metadata", {}) or {}
fields: Final = self._extract_metadata_fields(metadata, credentials)
# the proxy header fan-out mirrors one value into both keys, and LangSmith
# rejects the whole ingest batch when run-body session_id is not an
# existing tracer-session uuid
if fields["session_id"] == fields["trace_id"]:
fields["session_id"] = None
verbose_logger.debug(
"Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
)

4
litellm/integrations/otel/__init__.py
View file

@ -49,6 +49,7 @@ from litellm.integrations.otel.model.semconv import (
Error,
GenAI,
GenAIOperation,
GenAIOutputType,
GenAIProvider,
JsonRpc,
LiteLLM,
@ -60,6 +61,7 @@ from litellm.integrations.otel.model.semconv import (
RpcSystem,
Server,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.spans import (
@ -84,6 +86,7 @@ __all__ = [
"Error",
"GenAI",
"GenAIOperation",
"GenAIOutputType",
"GenAIProvider",
"GuardrailSpanData",
"JsonRpc",
@ -116,6 +119,7 @@ __all__ = [
"is_otel_v2_enabled",
"promoted_baggage",
"resolve_operation",
"resolve_output_type",
"resolve_provider",
"span_role_for_service",
"validate_registry",

6
litellm/integrations/otel/emitter.py
View file

@ -156,6 +156,12 @@ class SpanEmitter:
links=list(links) if links else None,
)
def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
"""Register a span emitted outside :meth:`emit` (the boundary-opened
LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
for the same ``(dedup_key, role)`` deduplicates against it."""
self._seen(dedup_key, role)
def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
"""Return True once a ``(dedup_key, role)`` pair has been emitted.

36
litellm/integrations/otel/logger.py
View file

@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
# ``pop`` is the dedup: this method runs from both the success and failure
# paths, and whichever fires first removes the carrier and closes the span.
carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
if carrier is None:
# A missing carrier does not always mean nothing happened: a team/key-scoped
# logger is a success/failure callback only, so ``pre_call`` never reaches it
# and no carrier exists. The payload plus the request-level provider-handoff
# stamp (``upstream_started``) is the affirmative signal of a real call; a
# gate rejection carries ``is_no_upstream_call`` and gets no span.
if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
return None
try:
return self._finish_carrier(carrier, call, end_time)
return self._finish_carrier(carrier, call, start_time, end_time)
finally:
# After the span has ended, so a release-triggered provider shutdown
# force-flushes it out rather than racing its enqueue.
@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
"""Remember an in-flight LLM call, evicting the oldest if over budget.
A call that opens but never closes (a stream that only fires stream
events) would linger otherwise; the evicted span is simply dropped
(never exported).
events) would linger otherwise. Eviction only drops the boundary carrier,
not the call: if that call later closes as a real completed call, it still
emits through the deferred branch in ``_close_llm_call`` (the same path a
team/key-scoped logger uses, since it never opens a carrier), deduplicated
by call id. Only a call that is evicted and never closes goes unexported.
"""
self._open_llm_calls[call_id] = carrier
if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
def _finish_carrier(
self,
carrier: _LLMCallSpan,
carrier: "_LLMCallSpan | None",
call: LLMCallEvent,
start_time: datetime | float | None,
end_time: datetime | float | None,
) -> Span | None:
payload: Final = call.payload
call_id: Final = call.call_id
if payload is None:
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Opened at the boundary but the payload never materialized — end
# it (named provisionally) so it isn't leaked as an open span.
# it (named provisionally) so it isn't leaked as an open span, and
# register the dedup marker so a later payload-carrying close for
# the same call id cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
carrier.span.end(end_time=to_ns(end_time))
return None
data: Final = LLMCallSpanData.from_standard_logging_payload(
@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
)
end_time_ns: Final = to_ns(end_time)
if carrier.span is not None:
if carrier is not None and carrier.span is not None:
# Born at the boundary: stamp attributes from the typed payload, set
# status, and end it. Its parent (the server span) was captured at
# creation from real ambient context.
# creation from real ambient context. Register the dedup marker so a
# second close for the same call id (success then failure on one
# logging object) cannot re-emit through the deferred branch.
self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
return carrier.span
# Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
SpanRole.LLM_CALL,
data,
parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
start_time_ns=carrier.start_time_ns,
start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
end_time_ns=end_time_ns,
tracer=route.tracer,
links=_request_trace_links(parent_ctx) if route.detached else None,

2
litellm/integrations/otel/mappers/genai.py
View file

@ -42,6 +42,7 @@ class GenAIMapper:
_LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
GenAI.OPERATION_NAME: lambda d: d.operation.value,
GenAI.PROVIDER_NAME: lambda d: d.provider or None,
GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
@ -65,6 +66,7 @@ class GenAIMapper:
Server.ADDRESS: lambda d: d.server.address if d.server else None,
Server.PORT: lambda d: d.server.port if d.server else None,
LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
LiteLLM.CALL_TYPE: lambda d: d.call_type,
# The provider/underlying model is only known once routing has picked a
# deployment, so it can't ride identity Baggage (seeded at auth, before
# routing) onto the boundary-born LLM span — stamp it directly here.

10
litellm/integrations/otel/model/config.py
View file

@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
WEAVE_OTEL = "weave_otel"
LEVO = "levo"
AGENTOPS = "agentops"
NEWRELIC = "newrelic"
class _OTelV2Flag(BaseSettings):
@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
"auto (Simple for console/in_memory, Batch otherwise)."
),
)
requires_headers: bool = Field(
default=False,
description=(
"Skip this exporter when no headers are resolved. For destinations "
"that reject unauthenticated exports (e.g. New Relic), a spec kept "
"only as the per-request credential-stamping target would otherwise "
"export keyless traffic and produce a 4xx for every span batch."
),
)
class OpenTelemetryV2Config(BaseSettings):

6
litellm/integrations/otel/model/metadata.py
View file

@ -203,6 +203,11 @@ class LLMCallEvent:
# True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
# the ``pre_call`` hook but never made an upstream call, so they get no span.
is_no_upstream_call: bool
# True once the request handed off to a provider (``pre_call`` stamped
# ``api_call_start_time``). The affirmative signal that an LLM call was
# actually attempted — router pre-call rejections, SDK failures before the
# provider handoff, and standalone guardrail runs all lack it.
upstream_started: bool
# A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
# span is renamed from the typed payload at close (``finish_span``); this only
# needs to be reasonable for a span that never gets closed (a leak).
@ -221,6 +226,7 @@ class LLMCallEvent:
dynamic_params=kwargs.get("standard_callback_dynamic_params"),
auth_metadata=auth_metadata(payload, kwargs),
is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
upstream_started=kwargs.get("api_call_start_time") is not None,
provisional_span_name=f"{operation.value} {model}".strip(),
time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
)

12
litellm/integrations/otel/model/payloads.py
View file

@ -15,8 +15,10 @@ from litellm.integrations.otel.model.metadata import (
)
from litellm.integrations.otel.model.semconv import (
GenAIOperation,
GenAIOutputType,
MCPMethod,
resolve_operation,
resolve_output_type,
resolve_provider,
)
from litellm.integrations.otel.model.utils import (
@ -310,6 +312,11 @@ class LLMCallSpanData:
choices_out: tuple[Mapping[str, object], ...] = ()
system_fingerprint: str | None = None
time_to_first_chunk_seconds: float | None = None
# The requested output modality, set only on the routes that pin one (image
# generation, speech, transcription, OCR), and the litellm route itself, which
# keeps routes the convention folds into one operation distinguishable.
output_type: GenAIOutputType | None = None
call_type: str | None = None
@classmethod
def from_standard_logging_payload(
@ -334,8 +341,9 @@ class LLMCallSpanData:
# otherwise the content-bearing mappers receive empty sequences and emit
# no prompt/response text.
finish_reasons: Final = _finish_reasons(choices_out)
call_type: Final = as_str(payload.get("call_type"))
return cls(
operation=resolve_operation(as_str(payload.get("call_type"))),
operation=resolve_operation(call_type),
provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
request_model=context.request_model,
response_model=context.response_model,
@ -358,6 +366,8 @@ class LLMCallSpanData:
choices_out=choices_out if capture_content else (),
system_fingerprint=as_str(response.get("system_fingerprint")),
time_to_first_chunk_seconds=time_to_first_chunk_seconds,
output_type=resolve_output_type(call_type),
call_type=call_type or None,
)

57
litellm/integrations/otel/model/semconv.py
View file

@ -3,7 +3,9 @@ Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anythin
without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
"""
from collections.abc import Mapping
from enum import Enum
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_logger
@ -30,6 +32,21 @@ class GenAIOperation(str, Enum):
EXECUTE_TOOL = "execute_tool" # MCP tool-call spans
LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
LITELLM_MODERATION = "litellm.moderation"
class GenAIOutputType(str, Enum):
"""Values for ``gen_ai.output.type``, the modality the client asked for.
It is what separates the inference routes that share ``generate_content``:
image generation requests ``image``, speech requests ``speech``, and
transcription and OCR both request ``text``.
"""
TEXT = "text"
JSON = "json"
IMAGE = "image"
SPEECH = "speech"
class GenAIProvider(str, Enum):
@ -258,6 +275,11 @@ class LiteLLM:
"""Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
CALL_ID: Final = "litellm.call_id"
# The litellm route that produced the call. Needed because the convention maps
# several routes onto one operation: transcription and OCR are both
# ``generate_content`` with a ``text`` output type, so this is the only thing
# that tells them apart.
CALL_TYPE: Final = "litellm.call_type"
COST_PREFIX: Final = "litellm.cost."
METADATA_PREFIX: Final = "litellm.metadata."
TEAM_ID: Final = "litellm.team.id"
@ -352,6 +374,16 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
"aembedding": GenAIOperation.EMBEDDINGS,
"responses": GenAIOperation.CHAT,
"aresponses": GenAIOperation.CHAT,
"image_generation": GenAIOperation.GENERATE_CONTENT,
"aimage_generation": GenAIOperation.GENERATE_CONTENT,
"moderation": GenAIOperation.LITELLM_MODERATION,
"amoderation": GenAIOperation.LITELLM_MODERATION,
"ocr": GenAIOperation.GENERATE_CONTENT,
"aocr": GenAIOperation.GENERATE_CONTENT,
"speech": GenAIOperation.GENERATE_CONTENT,
"aspeech": GenAIOperation.GENERATE_CONTENT,
"transcription": GenAIOperation.GENERATE_CONTENT,
"atranscription": GenAIOperation.GENERATE_CONTENT,
"call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
"vector_store_search": GenAIOperation.RETRIEVAL,
"avector_store_search": GenAIOperation.RETRIEVAL,
@ -385,6 +417,23 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
}
# litellm ``call_type`` -> ``gen_ai.output.type``. Only the call types whose route
# fixes the requested modality are listed; the attribute is conditionally required
# on a request that asks for an output format, so anything else is left unstamped.
_OUTPUT_TYPE_BY_CALL_TYPE: Final[Mapping[str, GenAIOutputType]] = MappingProxyType(
{
"image_generation": GenAIOutputType.IMAGE,
"aimage_generation": GenAIOutputType.IMAGE,
"speech": GenAIOutputType.SPEECH,
"aspeech": GenAIOutputType.SPEECH,
"transcription": GenAIOutputType.TEXT,
"atranscription": GenAIOutputType.TEXT,
"ocr": GenAIOutputType.TEXT,
"aocr": GenAIOutputType.TEXT,
}
)
def resolve_provider(custom_llm_provider: str | None) -> str:
"""Map a litellm provider string to a ``gen_ai.provider.name`` value.
@ -416,3 +465,11 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
GenAIOperation.CHAT.value,
)
return GenAIOperation.CHAT
def resolve_output_type(call_type: str | None) -> GenAIOutputType | None:
"""Map a litellm ``call_type`` to a ``gen_ai.output.type`` value, or ``None``
for a route that doesn't pin the output modality."""
if not call_type:
return None
return _OUTPUT_TYPE_BY_CALL_TYPE.get(call_type.lower())

2
litellm/integrations/otel/plumbing/providers.py
View file

@ -436,6 +436,8 @@ def build_tracer_provider(
# ``config._normalize`` guarantees at least one spec (it folds the top-level
# ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
for spec in config.exporters:
if spec.requires_headers and not spec.headers:
continue
exp = _exporter_from_spec(spec)
provider.add_span_processor(
_processor_for(

29
litellm/integrations/otel/plumbing/routing.py
View file

@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import (
get_tracer,
)
from litellm.integrations.otel.presets import (
dynamic_otlp_endpoint,
dynamic_otlp_headers,
project_routing_headers,
)
@ -129,7 +130,9 @@ class TenantTracerCache:
# thread-pool workers concurrently with the event loop, so cache
# updates, span counts, and retirement must be atomic.
self._lock: Final = threading.Lock()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
OrderedDict() # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
)
self._open_span_counts: dict[TracerProvider, int] = {} # mutable-ok: live refcount state
# Oldest-first so an overflow of draining providers sheds the stalest.
self._retired: OrderedDict[TracerProvider, None] = OrderedDict() # mutable-ok: draining evicted providers
@ -182,12 +185,16 @@ class TenantTracerCache:
project_headers: Final = self._project_headers(auth_metadata)
if not credential_headers and not project_headers:
return TenantRoute(tracer=default, detached=False)
# A fixed per-integration region endpoint (New Relic us/eu), never a
# caller-supplied host; ``None`` keeps the preset's own endpoint.
endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
cache_key: Final = (
tuple(sorted(credential_headers.items())),
tuple(sorted(project_headers.items())),
endpoint,
)
with self._lock:
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
evicted: Final = self._evicted_on_overflow_locked()
if evicted is not None:
@ -200,15 +207,16 @@ class TenantTracerCache:
def _cached_provider_locked(
self,
cache_key: tuple[_HeaderItems, _HeaderItems],
cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None,
) -> TracerProvider:
cached: Final = self._providers.get(cache_key)
if cached is not None:
self._providers.move_to_end(cache_key)
return cached
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
self._providers[cache_key] = built
return built
@ -257,6 +265,7 @@ class TenantTracerCache:
self,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> OpenTelemetryV2Config:
"""Clone the config, rewriting headers on the callback's own exporter.
@ -272,7 +281,8 @@ class TenantTracerCache:
``Authorization``), which must survive routing to a project.
"""
exporters: Final = [
self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
self._routed_exporter(spec, credential_headers, project_headers, endpoint)
for spec in self._config.exporters
]
return self._config.model_copy(update={"exporters": exporters})
@ -281,6 +291,7 @@ class TenantTracerCache:
spec: ExporterSpec,
credential_headers: Mapping[str, str],
project_headers: Mapping[str, str],
endpoint: str | None = None,
) -> ExporterSpec:
kind: Final = spec.kind.lower()
if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@ -291,4 +302,10 @@ class TenantTracerCache:
if project_headers and kind not in _GRPC_KINDS
else base
)
return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
update: Final = { # mutable-ok: model_copy(update=...) requires a plain dict
field: value
for field, value in (("headers", routed), ("endpoint", endpoint))
if (field == "headers" and routed != spec.headers)
or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
}
return spec if not update else spec.model_copy(update=update)

70
litellm/integrations/otel/presets/__init__.py
View file

@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
)
from litellm.integrations.otel.presets.langtrace import langtrace_preset
from litellm.integrations.otel.presets.levo import levo_preset
from litellm.integrations.otel.presets.newrelic import (
newrelic_dynamic_endpoint,
newrelic_dynamic_headers,
newrelic_preset,
)
from litellm.integrations.otel.presets.phoenix import (
phoenix_preset,
phoenix_project_headers,
@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
#: Callback name → preset. The ``Preset`` annotation makes mypy verify every
#: registered value matches the preset interface.
PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"weave_otel": weave_preset,
}
PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
{
"agentops": agentops_preset,
"arize": arize_preset,
"arize_phoenix": phoenix_preset,
"langfuse_otel": langfuse_preset,
"langtrace": langtrace_preset,
"levo": levo_preset,
"newrelic": newrelic_preset,
"weave_otel": weave_preset,
}
)
#: Callback name → per-request OTLP header builder (team/key multi-tenant
#: routing). Only integrations that support dynamic credentials appear here —
#: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
#: default tracer.
DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
MappingProxyType(
{
"arize": arize_dynamic_headers,
"langfuse_otel": langfuse_dynamic_headers,
"newrelic": newrelic_dynamic_headers,
"weave_otel": weave_dynamic_headers,
}
)
)
#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
#: destination host varies per tenant (from a fixed region table, never a
#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
#: authoritative.
DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
MappingProxyType(
{
"newrelic": newrelic_dynamic_endpoint,
}
)
)
#: Callback name → per-request *routing* header builder, sourced from the key/team
@ -98,17 +123,34 @@ def project_routing_headers(
return builder(auth_metadata)
def dynamic_otlp_endpoint(
callback_name: str | None,
dynamic_params: StandardCallbackDynamicParams | None,
) -> str | None:
"""Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
``None`` means "keep the preset's own endpoint".
"""
resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
if resolver is None or not dynamic_params:
return None
return resolver(dynamic_params)
__all__ = [
"DYNAMIC_ENDPOINT_BY_CALLBACK",
"DYNAMIC_HEADERS_BY_CALLBACK",
"PRESET_BY_CALLBACK",
"PROJECT_HEADERS_BY_CALLBACK",
"Preset",
"agentops_preset",
"arize_preset",
"dynamic_otlp_endpoint",
"dynamic_otlp_headers",
"langfuse_preset",
"langtrace_preset",
"levo_preset",
"newrelic_preset",
"phoenix_preset",
"project_routing_headers",
"weave_preset",

104
litellm/integrations/otel/presets/newrelic.py Normal file
View file

@ -0,0 +1,104 @@
"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from litellm._logging import verbose_logger
from litellm.integrations.otel.model.config import (
ExporterOwner,
ExporterSpec,
OpenTelemetryV2Config,
)
from litellm.integrations.otel.presets.utils import ensure_mappers
from litellm.types.utils import StandardCallbackDynamicParams
#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
#: region enum rather than a free-form endpoint, so callback vars can never
#: redirect telemetry to an arbitrary host.
NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
{
"us": "https://otlp.nr-data.net",
"eu": "https://otlp.eu01.nr-data.net",
}
)
_DEFAULT_REGION: Final = "us"
class _NewRelicSettings(BaseSettings):
model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
# The same env vars the agent-based integration documents; the key is the
# operator-level fallback for traffic without team credentials, the region
# picks that fallback's data center, and the record-content flag keeps its
# documented meaning when the OTel path replaces the agent.
license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
def newrelic_preset(
*,
config_overrides: OpenTelemetryV2Config | None = None,
) -> OpenTelemetryV2Config:
settings: Final = _NewRelicSettings()
base: Final = config_overrides or OpenTelemetryV2Config()
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
(settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
)
return base.model_copy(
update={
"exporters": [
*base.exporters,
ExporterSpec(
kind="otlp_http",
endpoint=endpoint,
headers=(f"api-key={settings.license_key}" if settings.license_key else None),
owner=ExporterOwner.NEWRELIC,
requires_headers=True,
),
],
# New Relic ingests the OTLP GenAI semantic conventions natively.
"mapper_names": ensure_mappers(base.mapper_names, "genai"),
**(
{"capture_message_content": ("span_only" if settings.record_content else "no_content")}
if settings.record_content is not None
else {}
),
}
)
def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
"""Per-request New Relic OTLP headers from team/key dynamic params."""
api_key: Final = params.get("newrelic_api_key")
return {header: value for header, value in (("api-key", api_key),) if value}
def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
"""Per-request OTLP endpoint for the team's ``newrelic_region``.
Always the team's own region endpoint, defaulting to US when the team left
the region unset. It never falls through to the preset's endpoint, which
follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
ingest key must not inherit the operator's region and have its US-account
spans rejected by an EU-configured default (or vice versa). An unknown
region likewise resolves to the documented US default rather than a guess.
"""
region: Final = params.get("newrelic_region")
default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
if not region:
return default_endpoint
endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
if endpoint is None:
verbose_logger.warning(
"New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
region,
", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
)
return default_endpoint
return endpoint

53
litellm/integrations/prometheus.py
View file

@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import *
from litellm.types.integrations.prometheus import (
_sanitize_prometheus_label_name,
_sanitize_prometheus_label_value,
validate_prometheus_deployment_and_latency_caller_identity,
)
from litellm.types.utils import (
StandardLoggingGuardrailInformation,
@ -96,7 +97,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
"""View a repository's prisma table through the pagination surface budget metrics need."""
return repository.table
return cast(
_PaginatedPrismaTable[_TableRowT],
repository.table, # cast-ok: prisma rows carry the budget columns the domain model declares
)
class _OrgBudgetRow(Protocol):
@ -172,6 +176,11 @@ class PrometheusLogger(CustomLogger):
try:
from prometheus_client import Counter, Gauge, Histogram
# Validate the caller-identity mode before any collector registers so an
# invalid value cannot leave partially-registered metrics behind in the
# process-global registry.
validate_prometheus_deployment_and_latency_caller_identity()
# Always initialize label_filters, even for non-premium users
self.label_filters = self._parse_prometheus_config()
@ -215,7 +224,9 @@ class PrometheusLogger(CustomLogger):
# request latency metrics
self.litellm_request_total_latency_metric = self._histogram_factory(
"litellm_request_total_latency_metric",
"Total latency (seconds) for a request to LiteLLM",
"End-to-end latency (seconds) for a request to LiteLLM Proxy Server, from the moment "
"the request reached the proxy through the end of processing -- includes "
"authentication, pre-call hooks, the LLM API call, and post-call processing",
labelnames=self.get_labels_for_metric("litellm_request_total_latency_metric"),
buckets=self.latency_buckets,
)
@ -458,7 +469,8 @@ class PrometheusLogger(CustomLogger):
# Request queue time metric
self.litellm_request_queue_time_metric = self._histogram_factory(
"litellm_request_queue_time_seconds",
"Time spent in request queue before processing starts (seconds)",
"Time (seconds) from request arrival at the proxy to the start of pre-call "
"processing -- includes authentication and any ASGI-level queueing",
labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
buckets=self.latency_buckets,
)
@ -2078,27 +2090,37 @@ class PrometheusLogger(CustomLogger):
_labels,
)
# total request latency
# request queue time (time from arrival to processing start) -- read first so
# it can be folded into the total-latency metric below. start_time/end_time
# only span from after auth completes, so without this the "total" latency
# metric silently excludes auth and pre-call hook time.
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
# total request latency: true end-to-end, from request arrival (queue_time_seconds,
# when available) through the end of processing.
total_time_seconds: Final = self._safe_duration_seconds(
start_time=start_time,
end_time=end_time,
)
if total_time_seconds is not None:
_observed_total_time_seconds: Final = (
total_time_seconds + queue_time_seconds
if queue_time_seconds is not None and queue_time_seconds >= 0
else total_time_seconds
)
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"),
enum_values=enum_values,
label_context=label_context,
)
self.litellm_request_total_latency_metric.labels(**_labels).observe(total_time_seconds)
self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds)
self._track_end_user_metric_series(
self.litellm_request_total_latency_metric,
"litellm_request_total_latency_metric",
_labels,
)
# request queue time (time from arrival to processing start)
_litellm_params: Final = kwargs.get("litellm_params", {}) or {}
queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
if queue_time_seconds is not None and queue_time_seconds >= 0:
_labels = prometheus_label_factory(
supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_queue_time_seconds"),
@ -2449,6 +2471,7 @@ class PrometheusLogger(CustomLogger):
else:
_metadata = {
"user_api_key_alias": getattr(_metadata_raw, "user_api_key_alias", None),
"user_api_key_user_email": getattr(_metadata_raw, "user_api_key_user_email", None),
"user_api_key_team_id": getattr(_metadata_raw, "user_api_key_team_id", None),
"user_api_key_team_alias": getattr(_metadata_raw, "user_api_key_team_alias", None),
"user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
@ -2471,6 +2494,17 @@ class PrometheusLogger(CustomLogger):
return getattr(user_api_key_auth, "key_alias", None)
return None
def _get_user_email() -> str | None:
val = _metadata.get("user_api_key_user_email")
if val is not None:
return val
val = _litellm_params_metadata.get("user_api_key_user_email")
if val is not None:
return val
if user_api_key_auth is not None:
return self._safe_get(user_api_key_auth, "user_email")
return None
def _get_team_id() -> str | None:
val = _metadata.get("user_api_key_team_id")
if val is not None:
@ -2506,6 +2540,7 @@ class PrometheusLogger(CustomLogger):
return {
"api_key_alias": _get_api_key_alias(),
"user_email": _get_user_email(),
"team": _get_team_id(),
"team_alias": _get_team_alias(),
"hashed_api_key": _get_hashed_api_key(),
@ -2563,6 +2598,7 @@ class PrometheusLogger(CustomLogger):
_metadata: Final = standard_logging_payload.get("metadata", {}) or {}
hashed_api_key: Final = fallback_values.get("hashed_api_key") or _metadata.get("user_api_key_hash")
api_key_alias: Final = fallback_values.get("api_key_alias") or _metadata.get("user_api_key_alias")
user_email: Final = fallback_values.get("user_email")
team: Final = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
team_alias: Final = fallback_values.get("team_alias") or _metadata.get("user_api_key_team_alias")
client_ip: Final = fallback_values.get("client_ip") or _metadata.get("requester_ip_address")
@ -2603,6 +2639,7 @@ class PrometheusLogger(CustomLogger):
requested_model=label_requested_model,
hashed_api_key=hashed_api_key,
api_key_alias=api_key_alias,
user_email=user_email,
team=team,
team_alias=team_alias,
tags=standard_logging_payload.get("request_tags", []),

99
litellm/integrations/s3_v2.py
View file

@ -11,6 +11,7 @@ import time
from collections.abc import Mapping
from datetime import datetime
from typing import Final, cast
from urllib.parse import quote
import litellm
from litellm._logging import print_verbose, verbose_logger
@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
)
def _build_object_url(self, s3_object_key: str) -> str:
"""
Build the exact URL that is both signed and sent, with the key percent-encoded once.
S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
the two signatures disagree (403 SignatureDoesNotMatch).
"""
encoded_key: Final = quote(s3_object_key, safe="/")
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
def _sse_headers(self) -> Mapping[str, str]:
candidates: Final = {
"x-amz-server-side-encryption": self.s3_server_side_encryption,
@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
import base64
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
aws_region_name=self.s3_region_name,
)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
url: Final = self._build_object_url(batch_logging_element.s3_object_key)
# Convert JSON to string
json_string: Final = safe_dumps(batch_logging_element.payload)
@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
"Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
**self._sse_headers(),
}
req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
data=prepped.body,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
# Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
request_url: Final = prepped.url or url
httpx_client: Final = _get_httpx_client(
params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
)
# Make the request with retry for transient S3 errors (500/503)
max_retries: Final = 3
for attempt in range(max_retries):
response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
response = httpx_client.put(url, data=json_string, headers=signed_headers)
if response.status_code in (500, 503) and attempt < max_retries - 1:
wait_time = 2**attempt # 1s, 2s
verbose_logger.warning(
@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
try:
import hashlib
import requests
from botocore.auth import S3SigV4Auth
from botocore.awsrequest import AWSRequest
except ImportError:
@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
# Prepare the URL
url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
if self.s3_endpoint_url and self.s3_bucket_name:
if self.s3_use_virtual_hosted_style:
# Virtual-hosted-style: bucket.endpoint/key
endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
else:
# Path-style: endpoint/bucket/key
url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
url: Final = self._build_object_url(s3_object_key)
# Prepare the request for GET operation
# For GET requests, we need x-amz-content-sha256 with hash of empty string
@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
headers: Final = {
"x-amz-content-sha256": empty_string_hash,
}
req: Final = requests.Request("GET", url, headers=headers)
prepped: Final = req.prepare()
# Sign the request
aws_request: Final = AWSRequest(
method=prepped.method,
url=prepped.url,
headers=prepped.headers,
)
aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
# Prepare the signed headers
signed_headers: Final = dict(aws_request.headers.items())
request_url: Final = prepped.url or url
response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
if response.status_code != 200:
verbose_logger.exception("S3 object not found, saw response=", response.text)

53
litellm/integrations/websearch_interception/ARCHITECTURE.md
View file

@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
---
## Loop Ceiling
One intercepted request can chain several follow-up model calls, since the model often searches again after
reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
```yaml
litellm_settings:
websearch_interception_params:
enabled_providers: ["bedrock"]
max_agentic_loops: 5
```
Or per deployment, which wins over the feature-level setting:
```yaml
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
max_agentic_loops: 5
```
Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
carries it is ignored and one request can never drive an unbounded number of upstream model calls.
Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
validator down there would drop the deployment silently instead of refusing to start.
When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
rather than as a replacement.
Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
Both are tracked separately
---
## Streaming Support
WebSearch interception works transparently with both streaming and non-streaming requests.

22
litellm/integrations/websearch_interception/handler.py
View file

@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
self,
enabled_providers: list[LlmProviders | str] | None = None,
search_tool_name: str | None = None,
max_agentic_loops: int | None = None,
):
"""
Args:
@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
Default: None (all providers enabled)
search_tool_name: Name of search tool configured in router's search_tools.
If None, will attempt to use first available search tool.
max_agentic_loops: How many follow-up model calls one intercepted request
may chain before the loop is refused and the turn ends.
If None, LiteLLM's default of 3 applies.
"""
super().__init__()
# Convert enum values to strings for comparison
@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
else:
self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
self.search_tool_name = search_tool_name
self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
self._request_has_websearch = False # Track if current request has web search
@staticmethod
def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
"""
Reject loop ceilings the agentic loop cannot honor, at config load time.
"""
return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
async def try_short_circuit_search(
self,
model: str,
@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: "my-perplexity-search"
max_agentic_loops: 5
Usage:
config = litellm_settings.get("websearch_interception_params", {})
@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# Extract parameters from config
enabled_providers_str: Final = config.get("enabled_providers", None)
search_tool_name: Final = config.get("search_tool_name", None)
max_agentic_loops: Final = config.get("max_agentic_loops", None)
# Convert string provider names to LlmProviders enum values
enabled_providers: list[LlmProviders | str] | None = None
@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return cls(
enabled_providers=enabled_providers,
search_tool_name=search_tool_name,
max_agentic_loops=max_agentic_loops,
)
@staticmethod
@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag

313
litellm/interactions/background_cost_polling.py Normal file
View file

@ -0,0 +1,313 @@
"""
Cost tracking for background interactions.
A create request with ``background=true`` returns ``in_progress`` with no
usage block, and GET polls are deliberately never billed (billing them would
double-charge every poll; the GET response also does not echo ``background``,
so a poll cannot be told apart from a re-fetch of an already-billed
interaction). The create call is therefore the only place that can own
billing: it schedules a poll task that fetches the interaction until it
reaches a terminal status and logs the final usage as a single success event
attributed to the original request.
``requires_action`` is terminal for the interaction it names. The API has no
operation that resumes one: a caller answers a tool request by creating a new
interaction whose ``previous_interaction_id`` points at it, and that new
interaction bills itself. The paused interaction keeps the tokens it already
spent producing the tool request, so it is billed and settled where it stops
rather than polled until the timeout, which would both lose that usage and
hold its budget reservation open for the whole timeout window.
Deleting an interaction makes every subsequent poll fail, which would let a
caller retrieve the completed output themselves and then delete it before the
poll task settles, leaving the work unbilled and the budget reservation
refunded at the poll timeout. ``adelete`` therefore settles any pending poll
for the interaction before dispatching the delete: it fetches the current
state with the create's credentials, bills it if it is terminal with usage,
and releases the reservation otherwise. A settlement gate on the create's
logging object makes the poll task and the delete path mutually exclusive, so
the interaction is billed exactly once no matter who settles first.
"""
import asyncio
from collections.abc import Awaitable, Callable, Iterator, Mapping
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, TypeAlias
from litellm._logging import verbose_logger
from litellm.constants import (
BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.types.interactions import InteractionsAPIResponse
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
_TERMINAL_STATUSES: Final = frozenset(
{"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
)
_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
@dataclass(frozen=True, slots=True)
class BackgroundInteractionPollContext:
interaction_id: str
custom_llm_provider: str
logging_obj: "LiteLLMLoggingObj"
api_key: str | None = None
api_base: str | None = None
initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
from litellm.interactions import aget
return await aget(
interaction_id=context.interaction_id,
custom_llm_provider=context.custom_llm_provider,
api_key=context.api_key,
api_base=context.api_base,
**{
"no-log": True
}, # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
)
def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
elapsed = 0.0
interval = initial
while interval > 0 and elapsed + interval <= timeout:
yield interval
elapsed += interval
interval = min(interval * 2, maximum)
_SETTLED_KEY = "background_interaction_settled"
def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
return logging_obj.model_call_details.get(_SETTLED_KEY) is True
def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
"""
Exactly-once gate between the poll task and the delete-time settlement:
both run on the same event loop and neither awaits between reading and
setting the flag, so whichever claims first owns billing or release.
"""
if _is_settled(logging_obj):
return False
logging_obj.model_call_details[_SETTLED_KEY] = True # rebind-ok: both settlers must see the same settlement flag
return True
async def poll_and_log_background_interaction_cost(
context: BackgroundInteractionPollContext,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
last_seen_status: str | None = None
for interval in _poll_intervals(
initial=context.initial_interval_seconds,
maximum=context.max_interval_seconds,
timeout=context.timeout_seconds,
):
await asyncio.sleep(interval)
if _is_settled(context.logging_obj):
return
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # any fetch error must not kill the billing poll loop
verbose_logger.debug(
"Background interaction cost poll for %s failed, will retry: %s",
context.interaction_id,
e,
)
continue
last_seen_status = response.status
if response.status not in _TERMINAL_STATUSES:
continue
if not _claim_settlement(context.logging_obj):
return
if response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
else:
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
verbose_logger.error(
"Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
"the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
"will not be tracked",
context.interaction_id,
context.timeout_seconds,
last_seen_status,
)
else:
verbose_logger.warning(
"Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
context.interaction_id,
context.timeout_seconds,
)
await _release_open_budget_reservation(logging_obj=context.logging_obj)
async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
"""
The proxy keeps the pre-call budget reservation open for an in-progress
background interaction so concurrent creates cannot stack past the budget.
The completion success event reconciles it to the actual cost; when the
interaction terminates without billable usage (or polling gives up, or it
is deleted before settling), no such event fires, so whoever claims the
settlement must release the reservation here or the spend counters stay
pinned at the estimated cost.
"""
metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
budget_reservation = metadata.get("user_api_key_budget_reservation")
if not isinstance(budget_reservation, dict):
return
from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
try:
await release_budget_reservation(budget_reservation=budget_reservation)
except Exception: # noqa: BLE001 # a failed release must not crash the poll task; counters expire via TTL
verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
"""
Claiming the settlement makes the claimer solely responsible for the
reservation, and no one retries a claim that is already set. A billing
failure here must therefore release the reservation on its way out, or it
stays pinned at the estimated cost until the whole poll times out.
"""
try:
await logging_obj.async_log_background_interaction_completion(result=response)
except Exception:
await _release_open_budget_reservation(logging_obj=logging_obj)
raise
def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
"""
The single gate deciding whether a create's response gets a poll task.
The proxy's success callback defers releasing the budget reservation for
exactly these responses, on the promise that a poll task will settle them,
so a response one site accepts and the other refuses strands its
reservation on the spend counters with nothing left to reconcile it.
``queued`` belongs here alongside ``in_progress``. It is the API's
not-started-yet state, so it reaches a terminal status the same way and
needs polling for the same reason: nothing else in the proxy ever bills a
create that came back without usage, so a status missing from both this
set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
"""
return response.status in _POLLABLE_STATUSES and bool(response.id)
def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
"""
Whether a response arriving with no usage block is a normal outcome rather
than lost billing data. An interaction that is still running, or that
stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
has nothing to charge for and should not raise a cost-tracking alarm.
``completed`` and ``requires_action`` both mean the model produced output,
so a usage block is always expected with them. If one arrives without it
the charge for real work has been lost, which is precisely what the
proxy's cost-tracking alert exists to surface.
"""
return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
@dataclass(frozen=True, slots=True)
class _ActiveBackgroundPoll:
task: "asyncio.Task[None]"
context: BackgroundInteractionPollContext
_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {} # mutable-ok: asyncio needs strong refs to running poll tasks
def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is not None and entry.task is task:
del _ACTIVE_POLLS[interaction_id]
def maybe_schedule_background_interaction_cost_polling(
response: object,
create_kwargs: Mapping[str, object],
custom_llm_provider: str,
) -> "asyncio.Task[None] | None":
from litellm.litellm_core_utils.litellm_logging import Logging
if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
return None
if not isinstance(response, InteractionsAPIResponse):
return None
if not is_pollable_background_interaction(response):
return None
logging_obj = create_kwargs.get("litellm_logging_obj")
if not isinstance(logging_obj, Logging):
return None
try:
asyncio.get_running_loop()
except RuntimeError:
return None
api_key = create_kwargs.get("api_key")
api_base = create_kwargs.get("api_base")
context = BackgroundInteractionPollContext(
interaction_id=response.id,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
api_key=api_key if isinstance(api_key, str) else None,
api_base=api_base if isinstance(api_base, str) else None,
)
task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
_ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
task.add_done_callback(
lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
)
return task
async def maybe_settle_background_interaction_before_delete(
interaction_id: str,
fetch_interaction: FetchInteraction = _fetch_interaction,
) -> None:
entry = _ACTIVE_POLLS.get(interaction_id)
if entry is None:
return
context = entry.context
try:
response = await fetch_interaction(context)
except Exception as e: # noqa: BLE001 # unfetchable pre-delete state settles by releasing the reservation
verbose_logger.debug(
"Could not fetch background interaction %s before delete, releasing its reservation: %s",
interaction_id,
e,
)
if _claim_settlement(context.logging_obj):
await _release_open_budget_reservation(logging_obj=context.logging_obj)
return
if not _claim_settlement(context.logging_obj):
return
if response.status in _TERMINAL_STATUSES and response.usage is not None:
await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
return
await _release_open_budget_reservation(logging_obj=context.logging_obj)

12
litellm/interactions/main.py
View file

@ -40,6 +40,10 @@ from typing import Any, Final
import httpx
import litellm
from litellm.interactions.background_cost_polling import (
maybe_schedule_background_interaction_cost_polling,
maybe_settle_background_interaction_before_delete,
)
from litellm.interactions.http_handler import interactions_http_handler
from litellm.interactions.utils import (
InteractionsAPIRequestUtils,
@ -171,6 +175,12 @@ async def acreate(
else:
response = init_response
maybe_schedule_background_interaction_cost_polling(
response=response,
create_kwargs=kwargs,
custom_llm_provider=custom_llm_provider,
)
return response
except Exception as e:
raise litellm.exception_type(
@ -462,6 +472,8 @@ async def adelete(
loop: Final = asyncio.get_event_loop()
kwargs["adelete_interaction"] = True
await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id)
func: Final = partial(
delete,
interaction_id=interaction_id,

7
litellm/interactions/utils.py
View file

@ -47,6 +47,13 @@ def get_provider_interactions_api_config(
return GoogleAIStudioInteractionsConfig()
if provider in (LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value):
from litellm.llms.vertex_ai.interactions.transformation import (
VertexAIInteractionsConfig,
)
return VertexAIInteractionsConfig()
return None

59
litellm/litellm_core_utils/agentic_loop_settings.py Normal file
View file

@ -0,0 +1,59 @@
"""
Shared validation for the agentic loop ceiling.
``max_agentic_loops`` can be set in two places, and the two disagreed about
what a bad value means. The feature-level
``litellm_settings.websearch_interception_params.max_agentic_loops`` was
checked at config load, while a per-deployment
``model_list[].litellm_params.max_agentic_loops`` was passed straight through
to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
turning the tightest ceiling into the loosest one, and let a per-deployment
``"three"`` boot the proxy and then fail every request to that model.
Both settings now go through :func:`validated_max_agentic_loops`, which names
the field it rejected so the error says which line of the config to fix.
Anything that spells a whole number is still accepted, because the old
``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
"""
from typing import Final
DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
def _as_whole_number(value: object) -> int | None:
"""
Return ``value`` as an int when it spells a whole number, else ``None``.
``bool`` is excluded explicitly because it is an ``int`` subclass, so
``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
"""
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value) if value.is_integer() else None
if isinstance(value, str):
try:
return int(value.strip())
except ValueError:
return None
return None
def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
"""
Return ``max_agentic_loops`` as an int, or raise naming ``field``.
"""
if max_agentic_loops is None:
return None
ceiling: Final = _as_whole_number(max_agentic_loops)
if ceiling is None:
raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
if ceiling < 1:
raise ValueError(f"{field} must be at least 1, got {ceiling}")
return ceiling

9
litellm/litellm_core_utils/chat_completion_agentic_loop.py
View file

@ -5,6 +5,10 @@ from typing import Final, cast
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
return depth, max_loops, fingerprints

61
litellm/litellm_core_utils/core_helpers.py
View file

@ -58,6 +58,67 @@ def safe_divide(
return numerator / denominator
def _is_litellm_limit_rejection(exception: BaseException) -> bool:
from litellm.exceptions import RateLimitErrorCategory
litellm_limit_categories: Final = frozenset(
(RateLimitErrorCategory.LITELLM_RATE_LIMIT.value, RateLimitErrorCategory.LITELLM_BATCH_RATE_LIMIT.value)
)
return getattr(exception, "category", None) in litellm_limit_categories
def _is_proxy_rejection(exception: BaseException) -> bool:
if _is_litellm_limit_rejection(exception):
return True
try:
from starlette.exceptions import HTTPException
except ImportError:
return False
return isinstance(exception, HTTPException)
def _is_provider_originated(exception: BaseException) -> bool:
if _is_proxy_rejection(exception):
return False
if getattr(exception, "llm_provider", None):
return True
from litellm.llms.base_llm.chat.transformation import BaseLLMException
return isinstance(exception, BaseLLMException)
def is_expected_client_error(exception: BaseException | None) -> bool:
"""
True when the proxy itself rejected the request with an HTTP 4xx before any
provider call (bad key, budget, unknown model, guardrail). A 4xx returned by
a provider is an upstream or deployment problem, so it is never an expected
client error and keeps its traceback: a mapped litellm exception carries
``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
raise before mapping (the /v1/messages route surfaces it as-is) is one too.
The proxy's own limiters raise ``HTTPException`` subclasses that also carry
an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection, and
so does any exception whose unified rate-limit ``category`` names litellm's
own limiter (``BudgetExceededError`` is a plain ``Exception`` that the auth
handler decorates with the requested model's provider).
ProxyException stores the status on .code (as a str), HTTPException and
litellm exceptions on .status_code.
"""
if exception is None:
return False
if _is_provider_originated(exception):
return False
code: Final[object] = getattr(exception, "code", None)
status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
if status_code is None or isinstance(status_code, bool):
return False
try:
status: Final = int(str(status_code))
except ValueError:
return False
return 400 <= status < 500
def coerce_token_limit(value: object) -> int | None:
"""
Coerce a max_input_tokens / max_output_tokens value to an int, treating a

167
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -550,6 +550,13 @@ def _map_anthropic_exception(
llm_provider="anthropic",
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"AnthropicException - {error_str}",
llm_provider="anthropic",
model=model,
response=original_exception.response,
)
elif original_exception.status_code == 400 or original_exception.status_code == 413:
raise BadRequestError(
message=f"AnthropicException - {error_str}",
@ -755,12 +762,19 @@ def _map_openai_like_exception(
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 401 or original_exception.status_code == 403:
elif original_exception.status_code == 401:
raise AuthenticationError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
)
elif original_exception.status_code == 403:
raise PermissionDeniedError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=403),
)
elif original_exception.status_code == 400:
raise BadRequestError(
message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
@ -811,6 +825,24 @@ def _map_openai_like_exception(
)
_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
"""
Mantle reports context overflow as a structured validation error rather than
the plain-text patterns Bedrock itself uses, so it needs its own detection and a
message clients recognize as context overflow (litellm/litellm#36546).
"""
if "invalid_request_error" not in error_str and "validation_error" not in error_str:
return None
match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
if match is None:
return None
prompt_tokens, max_tokens = match.groups()
return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
def _map_bedrock_exception(
*,
model: str,
@ -821,6 +853,14 @@ def _map_bedrock_exception(
exception_provider: str,
extra_information: str,
) -> None:
if custom_llm_provider == "bedrock_mantle":
mantle_context_window_message = _get_bedrock_mantle_context_window_message(error_str)
if mantle_context_window_message is not None:
raise ContextWindowExceededError(
message=mantle_context_window_message,
model=model,
llm_provider=custom_llm_provider,
)
if (
"too many tokens" in error_str
or "expected maxLength:" in error_str
@ -2161,6 +2201,120 @@ def _map_openrouter_exception(
)
def _response_or_stub(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
response: Final = original_exception.response if hasattr(original_exception, "response") else None
if response is not None:
return response
return httpx.Response(
status_code=status_code, request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs")
)
def _map_exception_by_status(
*,
model: str,
original_exception: _ProviderHTTPException,
custom_llm_provider: str,
error_str: str,
exception_provider: str,
extra_information: str,
) -> None:
status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
if not isinstance(status_code, int) or status_code < 400:
return
message: Final = f"{exception_provider} - {error_str}"
response: Final = original_exception.response if hasattr(original_exception, "response") else None
match status_code:
case 401:
raise AuthenticationError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 403:
raise PermissionDeniedError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=_response_or_stub(original_exception, status_code=status_code),
litellm_debug_info=extra_information,
)
case 404:
raise NotFoundError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 408:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
)
case 429:
raise RateLimitError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case 500:
raise InternalServerError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 502:
raise BadGatewayError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 503:
raise ServiceUnavailableError(
message=message,
llm_provider=custom_llm_provider,
model=model,
response=response,
litellm_debug_info=extra_information,
)
case 504:
raise Timeout(
message=message,
model=model,
llm_provider=custom_llm_provider,
litellm_debug_info=extra_information,
exception_status_code=status_code,
)
case _ if status_code < 500:
raise BadRequestError(
message=message,
model=model,
llm_provider=custom_llm_provider,
response=response,
litellm_debug_info=extra_information,
)
case _:
raise APIError(
status_code=status_code,
message=message,
llm_provider=custom_llm_provider,
model=model,
request=original_exception.request if hasattr(original_exception, "request") else None,
litellm_debug_info=extra_information,
)
def exception_type(
model,
original_exception,
@ -2275,6 +2429,7 @@ def exception_type(
or custom_llm_provider == "custom_openai"
or custom_llm_provider in litellm.openai_compatible_providers
or custom_llm_provider == "mistral"
or custom_llm_provider == "runwayml"
):
_map_openai_exception(
model=model,
@ -2315,7 +2470,7 @@ def exception_type(
exception_provider=exception_provider,
extra_information=extra_information,
)
elif custom_llm_provider == "bedrock":
elif custom_llm_provider in ("bedrock", "bedrock_mantle"):
_map_bedrock_exception(
model=model,
original_exception=mappable_exception,
@ -2474,6 +2629,14 @@ def exception_type(
For unmapped exceptions - raise the exception with traceback - https://github.com/BerriAI/litellm/issues/4201
"""
exception_mapping_worked = True
_map_exception_by_status(
model=model,
original_exception=mappable_exception,
custom_llm_provider=custom_llm_provider,
error_str=error_str,
exception_provider=exception_provider,
extra_information=extra_information,
)
if hasattr(original_exception, "request"):
raise APIConnectionError(
message=f"{exception_provider} - {error_str}",

10
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -272,6 +272,14 @@ def get_llm_provider(
elif endpoint == "api.deepseek.com/v1":
custom_llm_provider = "deepseek"
dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
elif endpoint == "api.together.ai/v1" or endpoint == "api.together.xyz/v1":
custom_llm_provider = "together_ai"
dynamic_api_key = api_key or (
get_secret_str("TOGETHER_API_KEY")
or get_secret_str("TOGETHER_AI_API_KEY")
or get_secret_str("TOGETHERAI_API_KEY")
or get_secret_str("TOGETHER_AI_TOKEN")
)
elif endpoint == "ollama.com":
custom_llm_provider = "ollama"
dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
@ -707,7 +715,7 @@ def _get_openai_compatible_provider_info(
dynamic_api_key,
) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
elif custom_llm_provider == "together_ai":
api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.xyz/v1"
api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
dynamic_api_key = api_key or (
get_secret_str("TOGETHER_API_KEY")
or get_secret_str("TOGETHER_AI_API_KEY")

24
litellm/litellm_core_utils/get_provider_specific_headers.py
View file

@ -1,3 +1,4 @@
from collections.abc import Sequence
from typing import Final
from litellm.types.utils import ProviderSpecificHeader
@ -6,13 +7,17 @@ from litellm.types.utils import ProviderSpecificHeader
class ProviderSpecificHeaderUtils:
@staticmethod
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | None,
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
custom_llm_provider: str | None,
) -> dict:
"""
Get the provider specific headers for the given custom llm provider.
Supports comma-separated provider lists for headers that work across multiple providers.
Accepts either a single ProviderSpecificHeader or a sequence of them. Each entry
carries its own comma-separated provider list, so headers that are safe for several
providers and headers that are safe for exactly one can travel on the same request
without sharing a scope. Entries whose provider list does not contain
`custom_llm_provider` contribute nothing.
Returns:
Dict: The provider specific headers for the given custom llm provider
@ -20,10 +25,15 @@ class ProviderSpecificHeaderUtils:
if provider_specific_header is None or custom_llm_provider is None:
return {}
stored_providers: Final = provider_specific_header.get("custom_llm_provider", "")
provider_list: Final = [p.strip() for p in stored_providers.split(",")]
scoped_headers: Final = (
(provider_specific_header,) if isinstance(provider_specific_header, dict) else provider_specific_header
)
if custom_llm_provider in provider_list:
return provider_specific_header.get("extra_headers", {})
matched_headers: Final = {}
for scoped_header in scoped_headers:
stored_providers = scoped_header.get("custom_llm_provider", "")
provider_list = [p.strip() for p in stored_providers.split(",")]
if custom_llm_provider in provider_list:
matched_headers.update(scoped_header.get("extra_headers", {}))
return {}
return matched_headers

2
litellm/litellm_core_utils/get_supported_openai_params.py
View file

@ -172,7 +172,7 @@ def get_supported_openai_params(
if request_type == "embeddings":
return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "together_ai":
return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
elif custom_llm_provider == "databricks":
if request_type == "chat_completion":
return litellm.DatabricksConfig().get_supported_openai_params(model=model)

32
litellm/litellm_core_utils/initialize_dynamic_callback_params.py
View file

@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
# Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
_supported_callback_params: Final = [
_supported_callback_params: Final[tuple[str, ...]] = (
"langfuse_public_key",
"langfuse_secret",
"langfuse_secret_key",
@ -72,8 +72,10 @@ _supported_callback_params: Final = [
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
"turn_off_message_logging",
]
)
_request_blocked_callback_params: Final = frozenset(
{
@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset(
"dd_site",
"dd_agent_host",
"dd_agent_port",
"newrelic_api_key",
"newrelic_region",
}
)
# Request-blocked params that must still reach ``standard_callback_dynamic_params``
# when the proxy itself stamped them from admin-configured team/key callback
# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads
# ``standard_callback_dynamic_params``, so without this overlay a blocked param
# could never drive routing at all.
_trusted_overlay_callback_params: Final = frozenset(
{
"newrelic_api_key",
"newrelic_region",
}
)
@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params(
if param in kwargs:
_param_value = kwargs.get(param)
validate_no_callback_env_reference(param, _param_value, source="request body")
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
for param in _supported_callback_params:
@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params(
if param not in standard_callback_dynamic_params and param in metadata:
_param_value = metadata.get(param)
validate_no_callback_env_reference(param, _param_value, source=slot_label)
standard_callback_dynamic_params[param] = _param_value
standard_callback_dynamic_params[param] = ( # pyright: ignore[reportGeneralTypeIssues] # several supported params predate their StandardCallbackDynamicParams fields
_param_value
)
for param, trusted_value in get_trusted_callback_params(kwargs):
if param in _trusted_overlay_callback_params:
standard_callback_dynamic_params[param] = trusted_value
return standard_callback_dynamic_params

97
litellm/litellm_core_utils/json_fragment_accumulator.py Normal file
View file

@ -0,0 +1,97 @@
import json
from typing import Final, cast # noqa: TID251 # raw_decode returns tuple[Any, int]; no cast-free unpack
class JSONFragmentAccumulator:
"""
Buffers a JSON value that arrives piecemeal over a stream (SSE data split
across TCP packets, one shard per network read, etc) without the O(n^2)
cost of repeated `buffer += fragment` string concatenation, and without
the O(n^2) cost of re-copying the unconsumed remainder on every peeled
value when one payload holds many concatenated JSON values.
Fragments are appended to a list in O(1). The buffer is only rebuilt into
a single string, and only decoded, when a caller asks for a value via
`pop_next_value`, and `could_close_json` lets callers skip that rebuild
entirely for fragments that plainly cannot close a JSON value yet. Once
rebuilt, consumed values are dropped by advancing a cursor rather than
slicing a new string, so draining N concatenated values already sitting
in the buffer costs O(n) total, not O(n^2).
"""
def __init__(self) -> None:
self._chunks: list[str] = [] # mutable-ok: O(1) append; string concat would copy the buffer each time
self._buffer: str = (
"" # mutable-ok: lazily materialized join of _chunks, rebuilt only when _chunks is non-empty
)
self._offset: int = 0 # mutable-ok: cursor past already-consumed values; avoids re-slicing on every pop
self._could_close: bool = False # mutable-ok: cached heuristic; rescanning past fragments was itself O(n^2)
def __bool__(self) -> bool:
return bool(self._chunks) or self._offset < len(self._buffer)
def append(self, fragment: str) -> None:
self._chunks.append(fragment) # mutable-ok: see __init__
stripped: Final = fragment.rstrip()
if stripped:
self._could_close = stripped[-1] in ("}", "]") # mutable-ok: see __init__
def could_close_json(self) -> bool:
"""
Whether the buffer's logical last non-whitespace byte is "}" or "]",
i.e. whether a JSON value could plausibly be complete. Tracked
incrementally in `append` rather than rescanned here, so a run of
blank keepalive fragments (e.g. from a malformed upstream stream)
can't make this, or the join+parse it gates, cost O(n^2).
"""
return self._could_close
def _materialize(self) -> None:
if not self._chunks:
return
unconsumed: Final = self._buffer[self._offset :]
self._buffer = unconsumed + "".join(self._chunks) # mutable-ok: merge pending fragments, once per append batch
self._offset = 0 # mutable-ok: see __init__
self._chunks = [] # mutable-ok: see __init__
def pop_next_value(self) -> tuple[bool, object]:
"""
Attempt to decode one complete JSON value from the front of the
buffer. On success, advances a cursor past that value (keeping any
unconsumed tail, e.g. a second concatenated value, in place rather
than copying it) and returns (True, value). If the buffer is empty
or holds no complete value yet, it is left untouched and this
returns (False, None).
"""
self._materialize()
length: Final = len(self._buffer)
start = self._offset
while start < length and self._buffer[start].isspace():
start += 1
if start >= length:
self._offset = start # mutable-ok: see __init__
return False, None
decoder: Final = json.JSONDecoder()
try:
raw_value: Final = decoder.raw_decode(self._buffer, start)
except json.JSONDecodeError:
return False, None
decoded, end_index = cast("tuple[object, int]", raw_value) # cast-ok: raw_decode returns tuple[Any, int]
self._offset = end_index # mutable-ok: see __init__
if self._offset >= len(self._buffer):
self._buffer = "" # mutable-ok: see __init__
self._offset = 0 # mutable-ok: see __init__
self._could_close = False # mutable-ok: buffer is empty, nothing can close
return True, decoded
def snapshot(self) -> str:
self._materialize()
return self._buffer[self._offset :]
def set(self, value: str) -> None:
"""Replace the buffer's contents with a single fragment."""
self._chunks = [] # mutable-ok: see __init__
self._buffer = value # mutable-ok: see __init__
self._offset = 0 # mutable-ok: see __init__
stripped: Final = value.rstrip()
self._could_close = bool(stripped) and stripped[-1] in ("}", "]") # mutable-ok: see __init__

228
litellm/litellm_core_utils/litellm_logging.py
View file

@ -62,7 +62,7 @@ from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.deepeval.deepeval import DeepEvalLogger
from litellm.integrations.mlflow import MlflowLogger
from litellm.integrations.sqs import SQSLogger
from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
cost_breakdown_with_guardrail,
@ -71,6 +71,9 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
InteractionsUsageObjectTransformation,
)
from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
from litellm.litellm_core_utils.redact_messages import (
@ -83,6 +86,10 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.responses.utils import ResponseAPILoggingUtils
from litellm.types.agents import LiteLLMSendMessageResponse
from litellm.types.containers.main import ContainerObject
from litellm.types.interactions import (
InteractionsAPIResponse,
InteractionsAPIStreamingResponse,
)
from litellm.types.llms.openai import (
AllMessageValues,
Batch,
@ -2145,6 +2152,11 @@ class Logging(LiteLLMLoggingBaseClass):
or isinstance(logging_result, OpenAIModerationResponse)
or isinstance(logging_result, OCRResponse) # OCR
or isinstance(logging_result, SearchResponse) # Search API
or (
isinstance(logging_result, InteractionsAPIResponse)
and logging_result.usage is not None
and self._is_interactions_create_call_type()
)
or isinstance(logging_result, dict)
and logging_result.get("object") == "vector_store.search_results.page"
or isinstance(logging_result, dict)
@ -2157,6 +2169,87 @@ class Logging(LiteLLMLoggingBaseClass):
return True
return False
def _is_interactions_create_call_type(self) -> bool:
"""
Only interaction creation is billable. GET polls, deletes, and cancels
also return an ``InteractionsAPIResponse`` (with usage once completed),
so recognizing those would write spend on every poll of a background
interaction. The proxy sets ``call_type`` from its route_type
(``create_interaction``/``acreate_interaction``); the SDK sets it from
the decorated function name (``create``/``acreate``).
Recognition additionally requires a usage block (checked at the call
site): a ``background=true`` create returns ``in_progress`` without
usage, and billing it would write a $0 spend log under the interaction
id that collides with the row the background poll task writes once the
interaction completes (see
``litellm.interactions.background_cost_polling``).
"""
return self.call_type in (
CallTypes.create_interaction.value,
CallTypes.acreate_interaction.value,
"create",
"acreate",
)
async def async_log_background_interaction_completion(
self,
result: InteractionsAPIResponse,
) -> None:
"""
Log the terminal result of a background interaction as a fresh success
event. The create request already ran success logging for its
``in_progress`` response (no usage, so no cost was tracked); clearing
the dedup flags lets the completed result flow through cost calculation
and spend tracking exactly once, spanning create to completion.
The poll fetched this body through its own client call, which priced it
against a throwaway logging object holding none of this request's
deployment context: no ``model_info``, no router ``model_id``, no
deployment ``litellm_params``. Keeping that price would bill a
custom-priced deployment at the wrong rate, and it would also satisfy
the "already calculated" shortcut and skip repricing here, leaving the
cost breakdown at the zeros the usage-less create stamped and writing
those zeros to the spend log. Dropping it makes this event price the
settled body itself, against the deployment that served the create.
The same throwaway call stamped the deployment identity that travels
with the price, so ``model_id`` and ``litellm_model_name`` go with it.
Left in place they overwrite the create's real deployment with the
poll's empty one in the payload every logging integration reads.
"""
settled_hidden_params: Final = getattr(result, "_hidden_params", None)
if isinstance(settled_hidden_params, dict):
for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
settled_hidden_params.pop(poll_scoped_key, None)
self._reset_success_emission_dedupe()
await self.async_success_handler(result=result)
def _reset_success_emission_dedupe(self) -> None:
"""
Success callbacks dedupe per request, because the sync and async
handlers both fire on some paths and would otherwise report one call
twice. A settled background interaction is a genuinely second success
event on the same request, so every such marker has to be cleared or
the completion, the only event that carries usage and cost, is
discarded as a duplicate of the in-progress create.
"""
self.model_call_details.pop("has_logged_async_success", None)
litellm_params = self.model_call_details.get("litellm_params")
if not isinstance(litellm_params, dict):
return
metadata = litellm_params.get("metadata")
if not isinstance(metadata, dict):
return
otel_internal = metadata.get("_otel_internal")
if not isinstance(otel_internal, dict):
return
spans_logged = otel_internal.get("spans_logged")
if not isinstance(spans_logged, dict):
return
for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]:
del spans_logged[scope]
def _flush_passthrough_collected_chunks_helper(
self,
raw_bytes: list[bytes],
@ -2282,7 +2375,9 @@ class Logging(LiteLLMLoggingBaseClass):
is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None = None
complete_streaming_response: (
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
) = None
if "complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response = self._get_assembled_streaming_response(
@ -2768,14 +2863,14 @@ class Logging(LiteLLMLoggingBaseClass):
## BUILD COMPLETE STREAMED RESPONSE
if "async_complete_streaming_response" in self.model_call_details:
return # break out of this.
complete_streaming_response: Final[ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None] = (
self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
complete_streaming_response: Final[
ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
] = self._get_assembled_streaming_response(
result=result,
start_time=start_time,
end_time=end_time,
is_async=True,
streaming_chunks=self.streaming_chunks,
)
if complete_streaming_response is not None:
@ -3029,6 +3124,13 @@ class Logging(LiteLLMLoggingBaseClass):
if not hasattr(self, "model_call_details"):
self.model_call_details = {}
if (
self.model_call_details.get("log_event_type") == "failed_api_call"
and self.model_call_details.get("exception") is exception
and self.model_call_details.get("standard_logging_object") is not None
):
return start_time, self.model_call_details["end_time"]
self.model_call_details["log_event_type"] = "failed_api_call"
self.model_call_details["exception"] = exception
self.model_call_details["traceback_exception"] = (
@ -3558,7 +3660,7 @@ class Logging(LiteLLMLoggingBaseClass):
end_time: datetime.datetime,
is_async: bool,
streaming_chunks: list[object],
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
if self.stream is not True:
return None
if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
@ -3583,9 +3685,40 @@ class Logging(LiteLLMLoggingBaseClass):
),
)
return result.response
elif isinstance(result, InteractionsAPIStreamingResponse):
return self._assemble_completed_interaction_response(result)
else:
return None
@staticmethod
def _assemble_completed_interaction_response(
result: InteractionsAPIStreamingResponse,
) -> InteractionsAPIResponse | None:
"""
The Interactions API streaming iterator hands the terminal event to the
success handlers: the new schema (Api-Revision: 2026-05-20) emits
``interaction.completed`` carrying the full interaction object, the
legacy schema (2026-05-07) emits a chunk with ``status="completed"``
and usage on the chunk itself. Build the equivalent non-streaming
response so cost calculation and spend tracking see one shape.
"""
if result.event_type == "interaction.completed" and result.interaction is not None:
return InteractionsAPIResponse(**result.interaction)
if result.status == "completed":
return InteractionsAPIResponse(
**result.model_dump(
exclude={ # mutable-ok: pydantic types exclude as set[str], which a frozenset does not satisfy
"event_type",
"delta",
"index",
"step",
"interaction_id",
"interaction",
}
)
)
return None
def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
"""
Handles logging for Anthropic messages responses.
@ -4503,6 +4636,9 @@ def _init_custom_logger_compatible_class(
_in_memory_loggers.append(gitlab_logger)
return gitlab_logger
elif logging_integration == "newrelic":
_v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
if _v2 is not None:
return _v2
for callback in _in_memory_loggers:
if isinstance(callback, NewRelicLogger):
return callback
@ -4789,7 +4925,11 @@ def get_custom_logger_compatible_class(
if isinstance(callback, SMTPEmailLogger):
return callback
elif logging_integration == "newrelic":
from litellm.integrations.otel.logger import OpenTelemetryV2
for callback in _in_memory_loggers:
if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic":
return callback
if isinstance(callback, NewRelicLogger):
return callback
return None
@ -5085,6 +5225,8 @@ class StandardLoggingPayloadSetup:
elif isinstance(usage, dict):
if ResponseAPILoggingUtils._is_response_api_usage(usage):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
return Usage(**usage)
raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
@ -5111,6 +5253,8 @@ class StandardLoggingPayloadSetup:
if isinstance(_raw, dict):
if ResponseAPILoggingUtils._is_response_api_usage(_raw):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
return _raw
if isinstance(_raw, Usage):
return _raw.model_dump()
@ -5318,9 +5462,10 @@ class StandardLoggingPayloadSetup:
error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
_llm_provider_in_exception: Final = getattr(original_exception, "llm_provider", "")
# Get traceback information (first 100 lines)
traceback_info = traceback_str or ""
if original_exception:
if original_exception and (
litellm.log_client_error_tracebacks or not is_expected_client_error(original_exception)
):
tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
if tb:
tb_lines: Final = traceback.format_tb(tb)
@ -5615,6 +5760,37 @@ def _extract_response_obj_and_hidden_params(
return response_obj, hidden_params
def _autorouter_savings_for_payload(
request_metadata: Mapping[str, object],
model: str | None,
custom_llm_provider: str | None,
model_id: str | None,
usage_object: Mapping[str, object] | None,
cost_breakdown: Mapping[str, object] | None,
) -> float | None:
"""The auto-router savings figure for the payload, or ``None`` when there is none.
Lazy proxy import: the savings module lives with the spend trackers that own the
math, and SDK-only installs have no proxy package to import.
"""
try:
from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload
except Exception: # noqa: BLE001 # SDK-only install: no savings driver to run
return None
try:
return autorouter_savings_for_logging_payload(
request_metadata=request_metadata,
model=model,
custom_llm_provider=custom_llm_provider,
model_id=model_id,
usage_object=usage_object,
cost_breakdown=cost_breakdown,
)
except Exception as e: # noqa: BLE001 # a savings figure must never fail request logging
verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
return None
def get_standard_logging_object_payload(
kwargs: dict | None,
init_response_obj: Any | BaseModel | dict,
@ -5762,16 +5938,30 @@ def get_standard_logging_object_payload(
response_model_name = final_response_obj.get("model")
# For Azure Model Router, preserve the actual model in the top-level standard
# logging payload only when the user has opted in.
# logging payload.
from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
requested_model: Final = kwargs.get("model")
if (
isinstance(requested_model, str)
and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower())
stamped_selected_model: Final = AzureFoundryModelInfo.get_model_router_selected_model(hidden_params)
if stamped_selected_model is not None:
model_name = stamped_selected_model
elif (
AzureFoundryModelInfo.is_model_router_call(model=requested_model, hidden_params=hidden_params)
and isinstance(response_model_name, str)
and response_model_name
):
model_name = response_model_name
request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
autorouter_savings: Final = _autorouter_savings_for_payload(
request_metadata=metadata,
model=model_name,
custom_llm_provider=custom_llm_provider,
model_id=_model_id,
usage_object=usage_dict,
cost_breakdown=request_cost_breakdown,
)
payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
id=str(id),
litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
@ -5802,7 +5992,8 @@ def get_standard_logging_object_payload(
metadata=clean_metadata,
cache_key=clean_hidden_params["cache_key"],
response_cost=response_cost,
cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
cost_breakdown=request_cost_breakdown,
autorouter_savings=autorouter_savings,
total_tokens=usage_dict.get("total_tokens", 0),
prompt_tokens=usage_dict.get("prompt_tokens", 0),
completion_tokens=usage_dict.get("completion_tokens", 0),
@ -5998,6 +6189,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
call_type="completion",
stream=False,
response_cost=response_cost,
autorouter_savings=None,
response_cost_failure_debug_info=None,
status="success",
total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT),

127
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,6 +1,9 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
TranscriptionUsageDurationObject,
TranscriptionUsageTokensObject,
@ -34,3 +37,127 @@ class TranscriptionUsageObjectTransformation:
),
)
return None
_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
{
"text": "text_tokens",
"audio": "audio_tokens",
"image": "image_tokens",
"video": "video_tokens",
"document": "text_tokens",
}
)
def _modality_field(entry: Mapping[str, Any]) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
def _token_count(value: object) -> int:
return value if isinstance(value, int) else 0
def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
for field in fields
}
)
def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
return sum(
_token_count(entry.get("count"))
for entry in tuple(usage_object.get("grounding_tool_count") or ())
if isinstance(entry, Mapping) and entry.get("type") == "google_search" # pyright: ignore[reportUnnecessaryIsInstance] # provider JSON, not the empty tuple inferred from `or ()`
)
def _subtract_cached_from_input(
input_sums: Mapping[str, int],
cached_sums: Mapping[str, int],
total_cached_tokens: int,
) -> Mapping[str, int]:
if cached_sums:
return MappingProxyType(
{field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()}
)
if total_cached_tokens and "text_tokens" in input_sums:
return MappingProxyType(
{
**input_sums,
"text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens),
}
)
return input_sums
class InteractionsUsageObjectTransformation:
"""
Maps the Google Interactions API usage block (total_input_tokens,
output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the
generic cost calculator and spend tracking can bill it.
"""
@staticmethod
def is_interactions_usage_object(usage_object: object) -> bool:
if not isinstance(usage_object, dict):
return False
if "prompt_tokens" in usage_object or "input_tokens" in usage_object:
return False
return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
)
cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
input_sums = _subtract_cached_from_input(
input_sums=_modality_token_sums(input_entries),
cached_sums=cached_sums,
total_cached_tokens=total_cached_tokens,
)
reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
usage_object.get("total_thought_tokens")
)
prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
usage_object.get("total_tool_use_tokens")
)
completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
web_search_requests = _google_search_query_count(usage_object)
prompt_tokens_details = (
PromptTokensDetailsWrapper(
cached_tokens=total_cached_tokens or None,
web_search_requests=web_search_requests or None,
**input_sums,
)
if input_sums or total_cached_tokens or web_search_requests
else None
)
completion_tokens_details = (
CompletionTokensDetailsWrapper(
reasoning_tokens=reasoning_tokens or None,
**output_sums,
)
if output_sums or reasoning_tokens
else None
)
return Usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
total_tokens=total_tokens,
prompt_tokens_details=prompt_tokens_details,
completion_tokens_details=completion_tokens_details,
cache_read_input_tokens=total_cached_tokens or None,
)

1
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -1371,6 +1371,7 @@ class CostCalculatorUtils:
return fal_ai_image_cost_calculator(
model=model,
image_response=completion_response,
optional_params=optional_params,
)
elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
from litellm.llms.runwayml.cost_calculator import (

80
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -1,8 +1,88 @@
from collections.abc import Mapping
from typing import Final
import litellm
def _form_field_value(value: object) -> str:
if value is True:
return "true"
if value is False:
return "false"
return str(value)
def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
if isinstance(value, Mapping):
return tuple(
item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def _is_form_scalar(value: object) -> bool:
return value is not None and not isinstance(value, (Mapping, list, tuple))
def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
if isinstance(value, Mapping):
return tuple(
item
for subkey, subvalue in value.items()
for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue)
)
if isinstance(value, (list, tuple)):
if all(_is_form_scalar(entry) for entry in value):
serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry)))
return ((key, serialized_fields),) if serialized_fields else ()
return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry))
if value is None:
return ()
serialized: Final = _form_field_value(value)
if not serialized:
return ()
return ((key, serialized),)
def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
"""
Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed
multipart body, applying ``sources`` in order so a later source wins on a key collision
under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK
serializes them, so provider params reach a multipart request without handing the httpx
encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a
single field carrying a tuple value, which httpx emits as one repeated part per element, so
every element survives instead of collapsing to the last under ``dict.update``.
"""
return tuple(
pair
for source in sources
if source is not None
for top_key, top_value in source.items()
for pair in _flatten_form_data_field(top_key, top_value)
)
def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]:
"""
Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less
request is still sent as multipart/form-data, working around httpx downgrading a
file-less ``data=`` payload to application/x-www-form-urlencoded.
"""
return tuple(
(key, (None, serialized))
for top_key, top_value in data.items()
for key, serialized in _flatten_form_field(top_key, top_value)
)
def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None:
"""
Ensure that the extra_body sent in the request is safe, otherwise users will see this error

11
litellm/litellm_core_utils/logging_utils.py
View file

@ -3,6 +3,7 @@ import functools
import inspect
import re
import time
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
@ -268,6 +269,16 @@ def _set_duration_in_model_call_details(
verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e)
def speech_request_body(model: str, voice: str, optional_params: Mapping[str, object]) -> Mapping[str, object]:
"""Speech request body for telemetry, without the caller headers the provider SDKs
take as request kwargs rather than body fields."""
return { # mutable-ok: loggers isinstance-check the request body as a dict
"model": model,
"voice": voice,
**{key: value for key, value in optional_params.items() if key != "extra_headers"},
}
def track_llm_api_timing():
"""
Decorator to track LLM API call timing for both sync and async functions.

36
litellm/litellm_core_utils/logging_worker.py
View file

@ -5,7 +5,7 @@ import asyncio
import atexit
import contextvars
import logging
from collections.abc import Coroutine
from collections.abc import Coroutine, Iterator
from typing import Final
from typing_extensions import TypedDict
@ -61,6 +61,19 @@ class LoggingWorker:
# Register cleanup handler to flush remaining events on exit
atexit.register(self._flush_on_exit)
@staticmethod
def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
"""Pop every task still queued, without awaiting them, so they can be moved to another queue."""
def _pop_until_empty() -> Iterator[LoggingTask]:
while True:
try:
yield queue.get_nowait()
except asyncio.QueueEmpty:
return
return tuple(_pop_until_empty())
def _ensure_queue(self) -> None:
"""Initialize the queue if it doesn't exist or if event loop has changed."""
try:
@ -69,14 +82,27 @@ class LoggingWorker:
# No running loop, can't initialize
return
# Check if we need to reinitialize due to event loop change
# The queue, semaphore and worker task are all bound to the loop that created them. On a
# loop change we hand the still-pending tasks to a fresh queue instead of dropping them,
# so queued spend-logging coroutines are not silently discarded (and never left un-awaited).
if self._queue is not None and self._bound_loop is not current_loop:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
# Clear old state - these are bound to the old loop
self._queue = None
carried_over: Final = self._drain_pending(self._queue)
new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
for carried_task in carried_over:
new_queue.put_nowait(carried_task)
if carried_over:
verbose_logger.warning(
"LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
len(carried_over),
)
else:
verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
self._sem = None
self._worker_task = None
self._running_tasks.clear()
self._queue = new_queue
self._bound_loop = current_loop
return
if self._queue is None:
self._queue = asyncio.Queue(maxsize=self.max_queue_size)

58
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -28,8 +28,12 @@ from litellm.types.llms.openai import (
ChatCompletionAssistantMessage,
ChatCompletionFileObject,
ChatCompletionImageObject,
ChatCompletionReasoningItem,
ChatCompletionReasoningSummaryTextBlock,
ChatCompletionRedactedThinkingBlock,
ChatCompletionResponseMessage,
ChatCompletionTextObject,
ChatCompletionThinkingBlock,
ChatCompletionToolParam,
ChatCompletionUserMessage,
)
@ -466,6 +470,8 @@ def update_messages_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
for message in messages:
@ -504,6 +510,11 @@ def update_messages_with_model_file_ids(
unified_file_id = convert_b64_uid_to_unified_uid(file_id)
if "llm_output_file_id," in unified_file_id:
provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
if not provider_file_id and is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
provider_file_id = get_original_file_id(file_id)
file_object_file_field["file_id"] = provider_file_id or file_id
if format:
file_object_file_field["format"] = format
@ -531,6 +542,8 @@ def update_responses_input_with_model_file_ids(
from litellm.proxy.openai_files_endpoints.common_utils import (
_is_base64_encoded_unified_file_id,
convert_b64_uid_to_unified_uid,
get_original_file_id,
is_model_embedded_id,
)
if isinstance(input, str):
@ -574,6 +587,13 @@ def update_responses_input_with_model_file_ids(
updated_content_item = content_item.copy()
updated_content_item["file_id"] = provider_file_id
updated_content.append(updated_content_item)
elif is_model_embedded_id(file_id):
# `litellm:<raw_id>;model,<m>` encoding from the
# x-litellm-model upload path. Strip the wrapper
# so the provider sees its own ID.
updated_content_item = content_item.copy()
updated_content_item["file_id"] = get_original_file_id(file_id)
updated_content.append(updated_content_item)
else:
# Not a managed file, keep as-is
updated_content.append(content_item)
@ -1549,6 +1569,44 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
return None, message_content
def _readable_thinking_text(
block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
) -> str:
"""The text a chat model can read back, empty for redacted blocks and malformed ones."""
if block.get("type") != "thinking":
return ""
thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking") # cast-ok: narrowed by the type tag
return str(thinking or "")
def reasoning_content_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> str:
"""Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
Redacted blocks carry no readable text, so they contribute nothing.
"""
return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
def responses_reasoning_item_from_thinking_blocks(
thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> ChatCompletionReasoningItem | None:
"""Build a Responses API `reasoning` input item from Anthropic thinking blocks.
The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
did not mint itself, while an item without an id is always accepted.
"""
summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [ # mutable-ok: API message payload
ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
for block in thinking_blocks
if (text := _readable_thinking_text(block))
]
if not summary:
return None
return ChatCompletionReasoningItem(type="reasoning", summary=summary)
def _parse_content_for_reasoning(
message_text: str | None,
) -> tuple[str | None, str | None]:

47
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -16,6 +16,7 @@ import litellm.types
import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.types.files import get_file_extension_from_mime_type
@ -642,49 +643,6 @@ def claude_2_1_pt(
return prompt
### TOGETHER AI
def get_model_info(token, model):
try:
headers: Final = {"Authorization": f"Bearer {token}"}
client: Final = HTTPHandler(concurrent_limit=1)
response: Final = client.get("https://api.together.xyz/models/info", headers=headers)
if response.status_code == 200:
model_info: Final = response.json()
for m in model_info:
if m["name"].lower().strip() == model.strip():
return m["config"].get("prompt_format", None), m["config"].get("chat_template", None)
return None, None
else:
return None, None
except Exception: # safely fail a prompt template request
return None, None
## OLD TOGETHER AI FLOW
# def format_prompt_togetherai(messages, prompt_format, chat_template):
# if prompt_format is None:
# return default_pt(messages)
# human_prompt, assistant_prompt = prompt_format.split("{prompt}")
# if chat_template is not None:
# prompt = hf_chat_template(
# model=None, messages=messages, chat_template=chat_template
# )
# elif prompt_format is not None:
# prompt = custom_prompt(
# role_dict={},
# messages=messages,
# initial_prompt_value=human_prompt,
# final_prompt_value=assistant_prompt,
# )
# else:
# prompt = default_pt(messages)
# return prompt
### IBM Granite
@ -5383,12 +5341,13 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
return raw
if not isinstance(raw, str):
return {}
normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
from litellm.litellm_core_utils.prompt_templates.common_utils import (
parse_tool_call_arguments,
)
try:
parsed: Final = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
except ValueError as e:
verbose_logger.warning("Failed to parse tool call arguments: %s", e)
return {}

52
litellm/litellm_core_utils/ptu_pricing.py
View file

@ -8,7 +8,7 @@ router prices at zero serves its traffic for free.
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime, timezone
from datetime import date, datetime, time, timezone
from types import MappingProxyType
from typing import Final
@ -68,9 +68,17 @@ def _to_utc(parsed: datetime) -> datetime:
def _as_utc(value: object) -> datetime | None:
"""A model_info datetime as UTC, parsing an ISO string, else None."""
"""A model_info datetime as UTC, parsing an ISO string, else None.
An unquoted ``2027-01-01`` in config.yaml is loaded as a ``date``, not a string, and a
reservation bound that fails to parse takes the whole deployment out of PTU handling,
so the day is read as its opening midnight rather than discarded. ``datetime`` derives
from ``date``, so it has to be matched first.
"""
if isinstance(value, datetime):
return _to_utc(value)
if isinstance(value, date):
return datetime.combine(value, time.min, tzinfo=timezone.utc)
if not isinstance(value, str):
return None
try:
@ -84,6 +92,46 @@ def _named(reason: str, model_name: str | None) -> str:
return reason if model_name is None else f"PTU configuration on model '{model_name}' is invalid: {reason}"
def ptu_identity_error(
*, declared_id: str | None, taken: bool, current_id: str | None = None, model_name: str | None = None
) -> str | None:
"""Why this config-declared reservation cannot be identified, else None.
A deployment declared in config.yaml is otherwise keyed by a hash of its resolved
``litellm_params``, so rotating a credential or editing an endpoint mints a second
identity and the reservation is charged again under it. The flat cost is keyed by that
id, and a charge already written is never retracted, so the duplicate is permanent.
``current_id`` is what the deployment is keyed by today. Naming it is the difference
between an operator carrying their history forward and an operator inventing a fresh
id, which starts a second identity beside the charges already written.
"""
if not declared_id:
return _named(
"model_info.id is required when PTU fields are set. Without one the deployment is "
"identified by a hash of its litellm_params, so rotating a credential bills the "
"reservation a second time under the new identity. Set it to the id this deployment "
f"already uses, {current_id or 'shown by GET /model/info'}, so the flat cost already "
"written stays under one identity; any other value starts a second one",
model_name,
)
if taken:
return _named(
f"model_info.id '{declared_id}' is declared on more than one deployment. Each would key "
"the same flat-cost row, so one reservation would go unbilled",
model_name,
)
return None
PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to")
def declares_ptu(model_info: Mapping[str, object]) -> bool:
"""Whether any PTU field is set here, including one too malformed to charge."""
return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS)
def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
"""Why this PTU configuration cannot be honoured, else None.

73
litellm/litellm_core_utils/redact_messages.py
View file

@ -13,6 +13,7 @@ import inspect
from typing import TYPE_CHECKING, Any, Final
import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
@ -84,29 +85,31 @@ def _redact_tool_calls(tool_calls) -> None:
for tool_call in tool_calls:
function = getattr(tool_call, "function", None)
if function is not None and hasattr(function, "arguments"):
function.arguments = "redacted-by-litellm"
function.arguments = REDACTED_BY_LITELLM
def _redact_function_call(function_call) -> None:
"""Redact legacy assistant function_call arguments."""
if function_call is not None and hasattr(function_call, "arguments"):
function_call.arguments = "redacted-by-litellm"
function_call.arguments = REDACTED_BY_LITELLM
def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, litellm.Choices):
choice.message.content = "redacted-by-litellm"
if hasattr(choice.message, "reasoning_content"):
choice.message.reasoning_content = "redacted-by-litellm"
if choice.message.content is not None:
choice.message.content = REDACTED_BY_LITELLM
if getattr(choice.message, "reasoning_content", None) is not None:
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
elif isinstance(choice, litellm.utils.StreamingChoices):
choice.delta.content = "redacted-by-litellm"
if hasattr(choice.delta, "reasoning_content"):
choice.delta.reasoning_content = "redacted-by-litellm"
if choice.delta.content is not None:
choice.delta.content = REDACTED_BY_LITELLM
if getattr(choice.delta, "reasoning_content", None) is not None:
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
_redact_tool_calls(getattr(choice.delta, "tool_calls", None))
@ -116,23 +119,23 @@ def _redact_choice_content(choice):
def _redact_responses_api_output(output_items):
"""Helper to redact ResponsesAPIResponse output items."""
for output_item in output_items:
if hasattr(output_item, "text"):
output_item.text = "redacted-by-litellm"
if getattr(output_item, "text", None) is not None:
output_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "content") and isinstance(output_item.content, list):
for content_part in output_item.content:
if hasattr(content_part, "text"):
content_part.text = "redacted-by-litellm"
if getattr(content_part, "text", None) is not None:
content_part.text = REDACTED_BY_LITELLM
# Redact reasoning items in output array
if hasattr(output_item, "type") and output_item.type == "reasoning":
if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
for summary_item in output_item.summary:
if hasattr(summary_item, "text"):
summary_item.text = "redacted-by-litellm"
if getattr(summary_item, "text", None) is not None:
summary_item.text = REDACTED_BY_LITELLM
if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
output_item.arguments = "redacted-by-litellm"
output_item.arguments = REDACTED_BY_LITELLM
def _redact_responses_api_output_dict(output_items, redacted_str: str):
@ -141,17 +144,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
if not isinstance(output_item, dict):
continue
if "text" in output_item:
if output_item.get("text") is not None:
output_item["text"] = redacted_str
if isinstance(output_item.get("content"), list):
for content_item in output_item["content"]:
if isinstance(content_item, dict) and "text" in content_item:
if isinstance(content_item, dict) and content_item.get("text") is not None:
content_item["text"] = redacted_str
if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
for summary_item in output_item["summary"]:
if isinstance(summary_item, dict) and "text" in summary_item:
if isinstance(summary_item, dict) and summary_item.get("text") is not None:
summary_item["text"] = redacted_str
if output_item.get("type") == "function_call" and "arguments" in output_item:
@ -164,7 +167,7 @@ def _redact_standard_logging_object(model_call_details: dict):
if standard_logging_object is None:
return
redacted_str: Final = "redacted-by-litellm"
redacted_str: Final = REDACTED_BY_LITELLM
if standard_logging_object.get("messages") is not None:
standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@ -188,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}
def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
def _redact_tool_calls_dict(message: dict) -> None:
"""Redact tool call / function_call arguments in a dict-form message or delta."""
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list):
for tool_call in tool_calls:
if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
tool_call["function"]["arguments"] = redacted_str
tool_call["function"]["arguments"] = REDACTED_BY_LITELLM
function_call: Final = message.get("function_call")
if isinstance(function_call, dict) and "arguments" in function_call:
function_call["arguments"] = redacted_str
function_call["arguments"] = REDACTED_BY_LITELLM
def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "reasoning_content" in choice["message"]:
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str
if choice["message"].get("reasoning_content") is not None:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
_redact_tool_calls_dict(choice["message"], redacted_str)
_redact_tool_calls_dict(choice["message"])
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "reasoning_content" in choice["delta"]:
if choice["delta"].get("content") is not None:
choice["delta"]["content"] = redacted_str
if choice["delta"].get("reasoning_content") is not None:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"], redacted_str)
_redact_tool_calls_dict(choice["delta"])
else:
_redact_choice_content(choice)
@ -235,7 +240,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
copy via redact_streaming_responses_for_custom_logger instead.
"""
# Redact model_call_details
model_call_details["messages"] = [{"role": "user", "content": "redacted-by-litellm"}]
model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
model_call_details["prompt"] = ""
model_call_details["input"] = ""
_redact_standard_logging_object(model_call_details)
@ -256,13 +261,13 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
or hasattr(result, "__anext__") # async generator
): # async iterator
# For async objects, return a simple redacted response without deepcopy
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
if not (
isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
or (isinstance(result, dict) and ("choices" in result or "output" in result))
):
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
_result: Final = copy.deepcopy(result)
if isinstance(_result, litellm.ModelResponse):
@ -273,11 +278,11 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
elif isinstance(_result, dict) and "choices" in _result:
# Handle dict representation of ModelResponse (e.g., from model_dump())
if _result.get("choices") is not None:
_redact_model_response_dict_choices(_result["choices"], "redacted-by-litellm")
_redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
redact_vertex_ai_metadata_from_logged_object(_result)
elif isinstance(_result, dict) and "output" in _result:
if isinstance(_result.get("output"), list):
_redact_responses_api_output_dict(_result["output"], "redacted-by-litellm")
_redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
elif isinstance(_result, litellm.ResponsesAPIResponse):
if hasattr(_result, "output"):
_redact_responses_api_output(_result.output)
@ -288,7 +293,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
if hasattr(_result, "data") and _result.data is not None:
_result.data = []
else:
return {"text": "redacted-by-litellm"}
return {"text": REDACTED_BY_LITELLM}
return _result

41
litellm/litellm_core_utils/streaming_handler.py
View file

@ -191,7 +191,7 @@ class CustomStreamWrapper:
custom_llm_provider: str | None = None,
stream_options=None,
make_call: Callable | None = None,
_response_headers: dict | None = None,
_response_headers: dict | httpx.Headers | None = None,
):
self.model = model
self.make_call = make_call
@ -2315,10 +2315,18 @@ class CustomStreamWrapper:
if self.logging_obj is None or not self.chunks:
return
try:
partial_response: Final = litellm.stream_chunk_builder(chunks=self.chunks)
partial_response: Final = litellm.stream_chunk_builder(
chunks=self.chunks,
messages=self.messages if isinstance(self.messages, list) else None,
)
if partial_response is None:
return
usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
if usage is None:
return
if self.model:
partial_response.model = self.model
backfill_missing_cache_usage_fields(usage)
self.logging_obj.model_call_details["combined_usage_object"] = usage
self.logging_obj.model_call_details["response_cost"] = (
self.logging_obj._response_cost_calculator(result=partial_response) or 0.0
@ -2439,6 +2447,35 @@ class CustomStreamWrapper:
return chunk
def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int:
for key in keys:
value = getattr(details, key, None)
if isinstance(value, int) and not isinstance(value, bool) and value:
return value
return 0
def backfill_missing_cache_usage_fields(usage: Usage) -> None:
"""Give partial-stream usage the same cache fields a complete stream reports.
Carries OpenAI-style ``prompt_tokens_details`` counts up to the Anthropic-style
top-level keys, defaulting to zero. It must carry the real count rather than a
flat zero: downstream readers treat these keys as authoritative once present and
skip their own normalization, so a zero here would overwrite a real cache read.
"""
details: Final = usage.prompt_tokens_details
if getattr(usage, "cache_read_input_tokens", None) is None:
usage.cache_read_input_tokens = _cache_token_count( # rebind-ok: in-place backfill is the contract
details, ("cached_tokens",)
)
if getattr(usage, "cache_creation_input_tokens", None) is None:
usage.cache_creation_input_tokens = _cache_token_count( # rebind-ok: in-place backfill is the contract
details, ("cache_write_tokens", "cache_creation_tokens")
)
if usage.prompt_tokens_details is None:
usage.prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=0) # rebind-ok: backfill in place
_TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper)

Some files were not shown because too many files have changed in this diff Show more