Merge pull request #38293 from BerriAI/litellm_internal_staging

chore(ci): promote internal staging to main

.circleci/config.yml

@@ -430,7 +430,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -504,7 +504,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -631,7 +631,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -651,126 +651,6 @@ jobs:
             - auth_ui_unit_tests_coverage.xml
             - auth_ui_unit_tests_coverage
 
-  proxy_behavior_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy management behavior tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_behavior \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  proxy_security_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy security tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_security_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  schema_migration_check:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      # An empty database; the test applies every committed migration itself.
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Check schema.prisma is in sync with committed migrations
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_migration_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
   litellm_router_testing: # Runs all tests with the "router" keyword
     docker:
       - *python312_image
@@ -858,7 +738,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -985,7 +865,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
                 -n 4 \
@@ -1030,7 +910,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1074,7 +954,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2 \
@@ -1120,7 +1000,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 --retries 3 --retry-delay 5"
@@ -1211,7 +1091,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1255,7 +1135,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1274,40 +1154,6 @@ jobs:
           paths:
             - search_coverage.xml
             - search_coverage
-  litellm_mapped_enterprise_tests:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-    resource_class: large
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - run:
-          name: Run enterprise tests
-          command: |
-            uv run --no-sync python -m prisma generate
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --junitxml=test-results/junit-enterprise.xml \
-                --durations=10 \
-                -n 4"
-          no_output_timeout: 15m
-      # Store test results
-      - store_test_results:
-          path: test-results
   batches_testing:
     docker:
       - *python312_image
@@ -1333,7 +1179,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1377,7 +1223,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1422,7 +1268,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1501,7 +1347,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 -n 4 \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1546,7 +1392,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1599,7 +1445,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 -n 2 \
                 --reruns 2 --reruns-delay 1"
@@ -3105,12 +2951,6 @@ workflows:
           filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
-      - proxy_behavior_tests:
-          filters: *main_branches
-      - proxy_security_tests:
-          filters: *main_branches
-      - schema_migration_check:
-          filters: *main_branches
       - build_docker_database_image:
           filters: *main_branches
       - e2e_ui_testing:
@@ -3167,8 +3007,6 @@ workflows:
           filters: *main_branches
       - search_testing:
           filters: *main_branches
-      - litellm_mapped_enterprise_tests:
-          filters: *main_branches
       - batches_testing:
           filters: *main_branches
       - litellm_utils_testing:
@@ -3191,7 +3029,6 @@ workflows:
             - guardrails_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing

.github/CODEOWNERS

@@ -1,6 +1,9 @@
 /ui/ @yuneng-berri @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

.github/ci-coverage-allowlist.yml

@@ -5,24 +5,21 @@ description: >-
 
 test_paths:
   - reason: >-
-      The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
-      directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
-      and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
-      names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
-      2026-08-20 by collecting the directory under each job's own selector: 118 tests across
-      these eight files are selected by none of them. Listed so the gap is a decision rather
-      than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
-      when tests/local_testing is ported off CircleCI, where the keyless part of this suite
-      belongs in a real job
+      What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
+      globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
+      not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
+      and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
+      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
+      run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
+      credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
+      test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
+      test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
+      split that porting tests/local_testing off CircleCI will force, not a job that is red by
+      construction
     paths:
-      - tests/local_testing/test_cache_preset_key.py
       - tests/local_testing/test_caching.py
-      - tests/local_testing/test_caching_handler.py
       - tests/local_testing/test_disk_cache_unit_tests.py
       - tests/local_testing/test_gcs_cache_unit_tests.py
-      - tests/local_testing/test_prompt_caching.py
-      - tests/local_testing/test_responses_stream_cache_keys.py
-      - tests/local_testing/test_unit_test_caching.py
   - reason: >-
       The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
       from a pull request; it needs a live gateway and provider credentials no PR job holds
@@ -92,17 +89,6 @@ test_paths:
       - tests/integration/sandbox/test_e2b_sandbox.py
       - tests/integration/test_oci_integration.py
       - tests/integration/test_oci_proxy_integration.py
-  - reason: >-
-      A unit test for the proxy-extras package that no job invokes, while the package's other
-      tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
-      and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
-      bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
-      this file requires. It also matches those keywords inside SQL comments, so two further
-      migrations are reported that are in fact fine. Wiring it up means deciding what to do about
-      the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
-      changed one breaks migrate deploy for existing installs
-    paths:
-      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
   - reason: >-

.github/pull_request_template.md

@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Include only the tiers that have caveats; drop the empty ones
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -134,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

.github/scripts/assert_ci_coverage.py

@@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
     )
 
 
+def _workflow_named_tokens() -> frozenset[str]:
+    """Test tokens a GitHub Actions job names directly.
+
+    A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
+    workflow names it, so the slice check has to credit those the same way the census does.
+    """
+    return _invoked_test_tokens(
+        scalar
+        for path in _config_files()
+        if path != CIRCLECI_CONFIG
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
 def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
     slices: Final = _slices()
+    named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
         path
         for path in _test_files()
@@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
         )
         for path in globbed
         if not allowlist.covers_test(path)
+        and not any(_token_covers(token, path) for token in named_by_workflow)
         and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
     )
 

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/workflows/_test-unit-base.yml

@@ -149,7 +149,7 @@ jobs:
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
@@ -161,7 +161,7 @@ jobs:
               --reruns-delay 1 \
               --dist="${DIST}" \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/image-scan.yml

@@ -23,6 +23,8 @@ on:
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -185,6 +187,35 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
 
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
+
   backend-image:
     name: backend-image
     runs-on: ubuntu-latest

.github/workflows/test-code-quality.yml

@@ -131,6 +131,9 @@ jobs:
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -67,6 +67,17 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-lint-
+
       - name: Clean Python cache
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-mcp.yml

@@ -60,4 +60,4 @@ jobs:
       - name: Run MCP tests
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

.github/workflows/test-postgres.yml

@@ -0,0 +1,145 @@
+name: "Postgres Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  postgres:
+    name: ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.job-timeout-minutes }}
+    permissions:
+      contents: read
+
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: proxy-behavior
+            test-path: "tests/proxy_behavior"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 25
+            job-timeout-minutes: 50
+
+          - shard: proxy-security
+            test-path: "tests/proxy_security_tests"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 15
+            job-timeout-minutes: 40
+
+          - shard: schema-migration
+            test-path: "tests/proxy_migration_tests"
+            seed: none
+            workers: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 45
+
+    env:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
+      - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 12
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Seed database schema
+        if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
+        timeout-minutes: 10
+        run: |
+          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+      - name: Run tests
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ matrix.timeout-minutes }}
+        env:
+          TEST_PATH: ${{ matrix.test-path }}
+          WORKERS: ${{ matrix.workers }}
+        run: |
+          if [ "${WORKERS}" = "0" ]; then
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
+          else
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
+          fi

.github/workflows/test-unit.yml

@@ -164,11 +164,12 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers
               tests/test_litellm/proxy/utils
-            workers: 2
+            workers: 4
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
@@ -195,11 +196,40 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: caching-local
+            artifact-name: caching-local
+            test-path: >-
+              tests/local_testing/test_cache_preset_key.py
+              tests/local_testing/test_caching_handler.py
+              tests/local_testing/test_prompt_caching.py
+              tests/local_testing/test_responses_stream_cache_keys.py
+              tests/local_testing/test_unit_test_caching.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
 
+          - shard: proxy-extras
+            artifact-name: proxy-extras
+            test-path: "tests/litellm-proxy-extras"
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: enterprise-package
+            artifact-name: enterprise-package
+            test-path: "tests/enterprise"
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
           - shard: responses-caching-types
             artifact-name: responses-caching-types
             test-path: >-

CLAUDE.md

@@ -37,13 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 
@@ -79,6 +80,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

basedpyright-code-budget.json

@@ -1,18 +1,18 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 18505
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2564
   },
   "reportAssignmentType": {
     "limit": 320
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 483
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 113
   },
   "reportConstantRedefinition": {
     "limit": 40
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 5976
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -45,7 +45,7 @@
     "limit": 35
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 34
   },
   "reportInvalidTypeVarUse": {
     "limit": 2
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5659
   },
   "reportMissingTypeArgument": {
-    "limit": 15555
+    "limit": 15504
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 1058
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1822
+    "limit": 1810
   },
   "reportRedeclaration": {
     "limit": 8
@@ -99,31 +99,31 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44655
+    "limit": 44530
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 38828
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19847
   },
   "reportUnknownVariableType": {
-    "limit": 30569
+    "limit": 30386
   },
   "reportUnnecessaryCast": {
     "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 697
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 833
   },
   "reportUntypedBaseClass": {
     "limit": 0

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -14,6 +14,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -351,7 +353,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.59"
+version = "0.1.60"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.59"
+version = "0.1.60"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm/values.yaml

@@ -428,9 +428,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

litellm/__init__.py

@@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+log_client_error_tracebacks: bool = False
 request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
@@ -1628,6 +1629,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -1801,6 +1805,9 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,

litellm/_lazy_imports_registry.py

@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -242,6 +243,7 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterResponsesAPIConfig",
     "BedrockMantleResponsesAPIConfig",
     "GoogleAIStudioInteractionsConfig",
+    "VertexAIInteractionsConfig",
     "OpenAIOSeriesConfig",
     "AnthropicSkillsConfig",
     "BaseSkillsAPIConfig",
@@ -740,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
@@ -977,6 +983,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.gemini.interactions.transformation",
         "GoogleAIStudioInteractionsConfig",
     ),
+    "VertexAIInteractionsConfig": (
+        ".llms.vertex_ai.interactions.transformation",
+        "VertexAIInteractionsConfig",
+    ),
     "OpenAIOSeriesConfig": (
         ".llms.openai.chat.o_series_transformation",
         "OpenAIOSeriesConfig",

litellm/a2a_protocol/card_resolver.py

@@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
 Extends the A2A SDK's card resolver to support multiple well-known paths.
 """
 
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
@@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
     return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
 
 
+_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
+    {
+        "jsonrpc": "JSONRPC",
+        "http+json": "HTTP+JSON",
+        "grpc": "GRPC",
+    }
+)
+
+_LEGACY_PROTOCOL_VERSION: Final = "0.3"
+
+
+def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
+    """
+    Canonicalize the supported interfaces of spec-adjacent agent cards.
+
+    Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
+    bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
+    case-sensitively against its uppercase TransportProtocol constants and fails
+    with "no compatible transports found." for spec-adjacent casings.
+
+    The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
+    payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
+    proto parsing rejects. A mis-cased binding fingerprints such a server, so its
+    declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
+    its v0.3 compat transport, which speaks that dialect.
+    """
+    normalized: Final = type(agent_card)()
+    normalized.CopyFrom(agent_card)
+    for interface in normalized.supported_interfaces:
+        canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
+        if canonical is None or canonical == interface.protocol_binding:
+            continue
+        interface.protocol_binding = canonical
+        interface.protocol_version = _LEGACY_PROTOCOL_VERSION
+    return normalized
+
+
 def get_agent_card_url(agent_card: "AgentCard") -> str | None:
     """Return the agent endpoint URL from the resolved SDK card."""
     url: Final = getattr(agent_card, "url", None)

litellm/a2a_protocol/main.py

@@ -73,6 +73,7 @@ except ImportError:
 from litellm.a2a_protocol.card_resolver import (
     LiteLLMA2ACardResolver,
     get_agent_card_url,
+    normalize_agent_card_interfaces,
 )
 from litellm.a2a_protocol.exception_mapping_utils import (
     handle_a2a_localhost_retry,
@@ -782,13 +783,17 @@ async def create_a2a_client(
     if extra_headers:
         verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
 
+    resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card: Final = normalize_agent_card_interfaces(
+        await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
+    )
+
     a2a_client: Final = await create_client(  # pyright: ignore[reportOptionalCall]
-        base_url,
+        agent_card,
         client_config=ClientConfig(  # pyright: ignore[reportOptionalCall]
             httpx_client=httpx_client,
             streaming=streaming,
         ),
-        resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
     )
     # Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
     # the configured httpx client and this agent's headers without excavating
@@ -799,9 +804,7 @@ async def create_a2a_client(
         if extra_headers
         else None
     )
-    agent_card: Final = getattr(a2a_client, "_card", None)
-    if agent_card is not None:
-        a2a_client._litellm_agent_card = agent_card
+    a2a_client._litellm_agent_card = agent_card
 
     verbose_logger.info("A2A client created for %s", base_url)
 

litellm/caching/redis_cluster_node_isolation.py

@@ -18,6 +18,14 @@ already does when one of its pooled connections errors), leaving every other nod
 connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
+
+redis-py 8.x fixed this upstream with gentler machinery than this override's
+``node.disconnect()`` (which also kills connections other coroutines are mid-operation
+on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
+killed connection): it marks in-use connections for reconnect only after their current
+operation completes, disconnects only the idle pooled ones, and defers reinitialization
+to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
+recovery API, the factory returns the base ``RedisCluster`` unmodified.
 """
 
 import asyncio
@@ -72,8 +80,16 @@ class _ClusterAttrs(Protocol):
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
 
 
-def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
-    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+def get_litellm_async_redis_cluster_class(
+    cluster_node_class: type | None = None,
+) -> type["_AsyncRedisClusterType"]:
+    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
+    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch
+    tears down the whole cluster client.
+
+    ``cluster_node_class`` exists for dependency injection in tests; production callers
+    leave it unset and the installed ``ClusterNode`` is used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -81,7 +97,10 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     """
     import redis
     from redis.asyncio.cluster import (
-        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+        ClusterNode as _AsyncClusterNode,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # same stale-stub gap as the import above
     )
     from redis.cluster import get_node_name
     from redis.commands import READ_COMMANDS
@@ -98,6 +117,15 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     from redis.exceptions import ConnectionError as _RedisConnectionError
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
+    node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    if hasattr(node_class, "update_active_connections_for_reconnect"):
+        verbose_logger.debug(
+            "redis-py %s recovers a node-level connection error per-connection upstream; "
+            "using the base RedisCluster without litellm's node-isolation override.",
+            redis.__version__,
+        )
+        return _BaseAsyncRedisCluster
+
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(
             "redis-py %s is not in the set this cluster-teardown-storm fix was verified "

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -21,6 +21,9 @@ from pydantic import BaseModel
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    responses_reasoning_item_from_thinking_blocks,
+)
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.bridges.completion_transformation import (
     CompletionTransformationBridge,
@@ -32,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import normalize_responses_api_stream_options
 from litellm.types.llms.openai import (
+    REASONING_EFFORT,
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
     ChatCompletionToolCallChunk,
@@ -85,6 +89,22 @@ def _get_reasoning_items(
     return []
 
 
+def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:  # mutable-ok: API message payload
+    """Reasoning input items for an assistant message.
+
+    Stored reasoning items win because they carry an id the Responses API minted; thinking
+    blocks are the fallback for turns that arrived over another API surface.
+    """
+    items: Final = _get_reasoning_items(msg)
+    stored: Final = [_reasoning_item_to_response_input(item) for item in items]  # mutable-ok: API message payload
+    if stored:
+        return stored
+    raw_blocks: Final = msg.get("thinking_blocks") or ()
+    blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks)  # cast-ok: untyped client json
+    from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
+    return [] if from_thinking is None else [dict(from_thinking)]  # mutable-ok: API message payload
+
+
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: str | None,
@@ -372,8 +392,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         )
                     )
             elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
-                for r_item in _get_reasoning_items(msg):
-                    input_items.append(_reasoning_item_to_response_input(r_item))
+                input_items.extend(_reasoning_input_items(msg))
+                if content:
+                    input_items.append(
+                        {  # mutable-ok: API message payload
+                            "type": "message",
+                            "role": "assistant",
+                            "content": self._convert_content_to_responses_format(content, "assistant"),
+                        }
+                    )
                 for tool_call in tool_calls:
                     function = tool_call.get("function")
                     custom = tool_call.get("custom")
@@ -400,15 +427,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         raise ValueError(f"tool call not supported: {tool_call}")
             elif content is not None:
                 if role == "assistant":
-                    for r_item in _get_reasoning_items(msg):
-                        input_items.append(_reasoning_item_to_response_input(r_item))
+                    input_items.extend(_reasoning_input_items(msg))
                 input_items.append(
-                    {
+                    {  # mutable-ok: API message payload
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(content, cast(str, role)),
                     }
                 )
+            elif role == "assistant":
+                input_items.extend(_reasoning_input_items(msg))
 
         return input_items, instructions
 
@@ -1086,22 +1114,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # If string is passed, map with optional summary based on flag/env var
-        if reasoning_effort == "none":
-            return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
-        elif reasoning_effort == "high":
-            return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
-        elif reasoning_effort == "xhigh":
-            return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
-        elif reasoning_effort == "medium":
+        if reasoning_effort in get_args(REASONING_EFFORT):
             return (
-                Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
-            )
-        elif reasoning_effort == "low":
-            return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
-        elif reasoning_effort == "minimal":
-            return (
-                Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
+                Reasoning(effort=reasoning_effort, summary="detailed")
+                if auto_summary_enabled
+                else Reasoning(effort=reasoning_effort)
             )
         return None
 

litellm/constants.py

@@ -48,6 +48,9 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Data URIs exceeding this are replaced with a size placeholder.
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
+REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
+# in-memory stand-in handed to provider converters for redacted arguments; never stored
+REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
 
 MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
 
@@ -146,6 +149,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-adaptive-router-model",
     "x-litellm-applied-guardrails",
     "x-litellm-guardrail-scan-id",
+    "x-litellm-cache-key",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -460,6 +464,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
 ]
 STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
+STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
+STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
     os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@@ -749,6 +755,7 @@ openai_compatible_endpoints: Final[list] = [
     "api.groq.com/openai/v1",
     "https://integrate.api.nvidia.com/v1",
     "api.deepseek.com/v1",
+    "api.together.ai/v1",
     "api.together.xyz/v1",
     "app.empower.dev/api/v1",
     "https://api.friendli.ai/serverless/v1",
@@ -1542,6 +1549,8 @@ SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BA
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
+RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3")))
+RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2"))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60))  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
@@ -1561,6 +1570,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
 # installations with large numbers of stale managed objects).
 _batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
 PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
+BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
+)
+BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
+)
+BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
+)
+_background_interaction_cost_polling_env: Final = os.getenv(
+    "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
+).lower()
+BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
 PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
 PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10))  # in seconds, increased from 10
 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

litellm/cost_calculator.py

@@ -19,6 +19,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
 from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
     TranscriptionUsageObjectTransformation,
 )
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -150,6 +151,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
     }
 )
 
+
 _SPEECH_CALL_TYPES: Final = frozenset(
     {
         CallTypes.speech.value,
@@ -912,6 +914,8 @@ def _get_usage_object(
                 usage_obj,
             )
         )
+    elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
+        return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
     elif isinstance(usage_obj, dict):
         return Usage(**usage_obj)
     elif isinstance(usage_obj, BaseModel):
@@ -1288,6 +1292,10 @@ def completion_cost(
                         )
                         if tr_usage is not None:
                             _usage = tr_usage.model_dump()
+                    elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
+                        _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
+                            _usage
+                        ).model_dump()
                     else:
                         _usage = _usage
 
@@ -1372,23 +1380,36 @@ def completion_cost(
                     if custom_pricing and litellm_logging_obj is not None:
                         _litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
                         if _litellm_params is not None:
-                            _metadata = _litellm_params.get("metadata", {}) or {}
-                            _video_model_info = _metadata.get("model_info", None)
+                            _video_model_info = next(
+                                (
+                                    model_info
+                                    for _metadata_key in ("metadata", "litellm_metadata")
+                                    if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
+                                    is not None
+                                ),
+                                None,
+                            )
 
                     usage_obj = getattr(completion_response, "usage", None)
                     duration_seconds: float | None = None
                     video_resolution: str | None = None
+                    provider_reported_cost: float | None = None
                     if completion_response is not None and usage_obj:
                         # Handle both dict and Pydantic Usage object
                         if isinstance(usage_obj, dict):
                             duration_seconds = usage_obj.get("duration_seconds", None)
                             _vr = usage_obj.get("video_resolution", None)
+                            provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
                         else:
                             duration_seconds = getattr(usage_obj, "duration_seconds", None)
                             _vr = getattr(usage_obj, "video_resolution", None)
+                            provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
                         if _vr is not None:
                             video_resolution = str(_vr).strip().lower()
 
+                        if _video_model_info is None and provider_reported_cost is not None:
+                            return float(provider_reported_cost)
+
                         if duration_seconds is not None:
                             # Calculate cost based on video duration using video-specific cost calculation
                             from litellm.llms.openai.cost_calculation import (

litellm/images/main.py

@@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
 from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
 from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -422,24 +423,32 @@ def image_generation(
                 aimg_generation=aimg_generation,
             )
         elif custom_llm_provider == "azure_ai":
-            from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+            from litellm.llms.azure_ai.common_utils import (
+                AzureFoundryModelInfo,
+                get_azure_ai_auth_headers,
+            )
 
             api_base = AzureFoundryModelInfo.get_api_base(api_base)
             api_key = AzureFoundryModelInfo.get_api_key(api_key)
             if extra_headers is not None:
                 optional_params["extra_headers"] = extra_headers
 
-            default_headers = {
+            caller_header_names = frozenset(name.lower() for name in headers)
+            caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
+            auth_headers = (
+                headers
+                if caller_set_auth
+                else get_azure_ai_auth_headers(
+                    api_key=api_key,
+                    litellm_params=litellm_params_dict,
+                    api_key_header="api-key",
+                )
+            )
+            request_headers: Final = {
                 "Content-Type": "application/json",
+                **auth_headers,
+                **headers,
             }
-            # Only add api-key header if api_key is not None
-            # Azure AD authentication will use Authorization header instead
-            if api_key is not None:
-                default_headers["api-key"] = api_key
-
-            for k, v in default_headers.items():
-                if k not in headers:
-                    headers[k] = v
 
             model_response = azure_chat_completions.image_generation(
                 model=model,
@@ -455,7 +464,7 @@ def image_generation(
                 api_version=api_version,
                 aimg_generation=aimg_generation,
                 client=client,
-                headers=headers,
+                headers=request_headers,
                 litellm_params=litellm_params_dict,
             )
         elif (
@@ -846,6 +855,18 @@ def image_edit(
             additional_drop_params=kwargs.get("additional_drop_params"),
         )
 
+        if (
+            custom_llm_provider == "openai"
+            or custom_llm_provider == "azure"
+            or custom_llm_provider in litellm.openai_compatible_providers
+        ):
+            image_edit_request_params.update(
+                flatten_form_field_values(
+                    non_default_params,
+                    extra_body if isinstance(extra_body, dict) else None,
+                )
+            )
+
         # Pre Call logging
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
@@ -995,6 +1016,9 @@ async def aimage_edit(
             response_format=response_format,
             size=size,
             user=user,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
             timeout=timeout,
             custom_llm_provider=custom_llm_provider,
             **kwargs,

litellm/integrations/callback_configs.json

@@ -294,12 +294,18 @@
     "id": "newrelic",
     "displayName": "New Relic",
     "logo": "newrelic.png",
-    "supports_key_team_logging": false,
+    "supports_key_team_logging": true,
     "dynamic_params": {
-      "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
+      "newrelic_api_key": {
+        "type": "password",
+        "ui_name": "New Relic Ingest License Key",
+        "description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
+        "required": false
+      },
+      "newrelic_region": {
         "type": "text",
-        "ui_name": "Record AI Content (default: true)",
-        "description": "Whether to record AI message content. Set to false to disable.",
+        "ui_name": "New Relic Region (us or eu)",
+        "description": "Data center region for this team's account. Defaults to us.",
         "required": false
       }
     },

litellm/integrations/compression_interception/handler.py

@@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
 
 import time
 import uuid
-from typing import Any, Final, cast
+from typing import Any, ClassVar, Final, cast
 
 from litellm._logging import verbose_logger
 from litellm.compression import compress
@@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
     4. Build typed rerun plan with tool_result blocks from the compressed cache.
     """
 
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
+
     def __init__(
         self,
         enabled: bool = True,

litellm/integrations/custom_logger.py

@@ -2,8 +2,8 @@
 #    On success, logs events to Promptlayer
 import re
 import traceback
-from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from collections.abc import AsyncGenerator, Mapping
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
 
@@ -60,6 +60,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
 
     enforces_request_content: bool = False
     """
@@ -292,6 +293,54 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
 
+    async def async_post_call_failure_deployment_hook(
+        self,
+        request_data: Mapping[str, Any],
+        exception: Exception,
+        call_type: CallTypes | None,
+        fallback_depth: int | None = None,
+    ) -> None:
+        """
+        Called once per failed deployment attempt - attempt 1, every retry, and
+        every fallback chain step - because the router re-invokes the wrapped
+        function on each attempt, re-entering this hook's call site fresh
+        every time.
+
+        This is a DEPLOYMENT-LEVEL signal, distinct from the REQUEST-LEVEL
+        ``async_log_failure_event``, which fires once per logical client
+        request behind a dedup gate. ``request_data`` is mostly this
+        attempt's own kwargs, with one exception: it omits
+        ``attempted_targets``, the router's own bookkeeping of which fallback
+        targets this request has already tried, since that one object *is*
+        shared by reference across every hop of the live fallback walk.
+
+        Pairs with ``async_pre_call_deployment_hook`` and
+        ``async_post_call_success_deployment_hook`` to complete the
+        pre-call/success/failure lifecycle for a single deployment attempt.
+
+        ``fallback_depth`` is best-effort: ``None`` on the first attempt and on
+        any call made without a ``Router`` (a bare SDK call has no fallback
+        chain to be at a depth in), ``1`` on the first fallback hop, ``2`` on
+        the second, and so on. It reflects ``Router``'s own internal fallback
+        bookkeeping (``kwargs["fallback_depth"]``), not a value this hook
+        computes or guarantees the shape of across versions. It tracks
+        fallback hops only, not retries within the same model group - a
+        retry-only failure (no fallback yet) also reports ``None``. If an
+        override predates this field it's simply never passed, rather than
+        raising - safe to leave off an override written before it existed.
+
+        ``exception`` is a same-class snapshot, not the exact object about to
+        be re-raised to the real caller: read it freely, but setting an
+        attribute on it (e.g. ``status_code``) has no effect on what the
+        caller actually receives.
+
+        Default: no-op. Opt in by overriding. Keep overrides fast - this
+        runs on the request's exception path, so a slow implementation
+        delays error propagation to the caller. The reported failure
+        duration is captured before this hook runs, so a slow override
+        doesn't inflate that metric, but the caller still waits for it.
+        """
+
     async def async_post_call_streaming_deployment_hook(
         self,
         request_data: dict,

litellm/integrations/langfuse/langfuse_otel.py

@@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
     LangfuseLLMObsOTELAttributes,
 )
 from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.types.integrations.langfuse_otel import (
     LangfuseSpanAttributes,
 )
@@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            safe_json_loads(arguments_str, default={})
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),

litellm/integrations/langsmith.py

@@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
         return outputs
 
     def _ensure_required_ids(self, data: dict, run_id: str | None):
+        resolved_id: Final = run_id or str(uuid.uuid4())
         if "id" not in data or data["id"] is None:
-            run_id = str(uuid.uuid4())
-            data["id"] = run_id
+            data["id"] = resolved_id
 
-        if "trace_id" not in data or data["trace_id"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["trace_id"] = run_id
+        # LangSmith rejects the whole ingest batch unless a root run's trace_id
+        # equals the run id embedded in the first segment of dotted_order
+        posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
+            "dotted_order" not in data or data["dotted_order"] is None
+        )
+        if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
+            data["trace_id"] = resolved_id
 
         if "dotted_order" not in data or data["dotted_order"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["dotted_order"] = self.make_dot_order(run_id=run_id)
+            data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
 
     def _prepare_log_data(
         self,
@@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
             metadata = _litellm_params.get("metadata", {}) or {}
 
             fields: Final = self._extract_metadata_fields(metadata, credentials)
+            # the proxy header fan-out mirrors one value into both keys, and LangSmith
+            # rejects the whole ingest batch when run-body session_id is not an
+            # existing tracer-session uuid
+            if fields["session_id"] == fields["trace_id"]:
+                fields["session_id"] = None
             verbose_logger.debug(
                 "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
             )

litellm/integrations/otel/emitter.py

@@ -156,6 +156,12 @@ class SpanEmitter:
             links=list(links) if links else None,
         )
 
+    def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
+        """Register a span emitted outside :meth:`emit` (the boundary-opened
+        LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
+        for the same ``(dedup_key, role)`` deduplicates against it."""
+        self._seen(dedup_key, role)
+
     def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
         """Return True once a ``(dedup_key, role)`` pair has been emitted.
 

litellm/integrations/otel/logger.py

@@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
         # ``pop`` is the dedup: this method runs from both the success and failure
         # paths, and whichever fires first removes the carrier and closes the span.
         carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
-        if carrier is None:
+        # A missing carrier does not always mean nothing happened: a team/key-scoped
+        # logger is a success/failure callback only, so ``pre_call`` never reaches it
+        # and no carrier exists. The payload plus the request-level provider-handoff
+        # stamp (``upstream_started``) is the affirmative signal of a real call; a
+        # gate rejection carries ``is_no_upstream_call`` and gets no span.
+        if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
             return None
         try:
-            return self._finish_carrier(carrier, call, end_time)
+            return self._finish_carrier(carrier, call, start_time, end_time)
         finally:
             # After the span has ended, so a release-triggered provider shutdown
             # force-flushes it out rather than racing its enqueue.
@@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
         """Remember an in-flight LLM call, evicting the oldest if over budget.
 
         A call that opens but never closes (a stream that only fires stream
-        events) would linger otherwise; the evicted span is simply dropped
-        (never exported).
+        events) would linger otherwise. Eviction only drops the boundary carrier,
+        not the call: if that call later closes as a real completed call, it still
+        emits through the deferred branch in ``_close_llm_call`` (the same path a
+        team/key-scoped logger uses, since it never opens a carrier), deduplicated
+        by call id. Only a call that is evicted and never closes goes unexported.
         """
         self._open_llm_calls[call_id] = carrier
         if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
 
     def _finish_carrier(
         self,
-        carrier: _LLMCallSpan,
+        carrier: "_LLMCallSpan | None",
         call: LLMCallEvent,
+        start_time: datetime | float | None,
         end_time: datetime | float | None,
     ) -> Span | None:
         payload: Final = call.payload
+        call_id: Final = call.call_id
         if payload is None:
-            if carrier.span is not None:
+            if carrier is not None and carrier.span is not None:
                 # Opened at the boundary but the payload never materialized — end
-                # it (named provisionally) so it isn't leaked as an open span.
+                # it (named provisionally) so it isn't leaked as an open span, and
+                # register the dedup marker so a later payload-carrying close for
+                # the same call id cannot re-emit through the deferred branch.
+                self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
                 carrier.span.end(end_time=to_ns(end_time))
             return None
         data: Final = LLMCallSpanData.from_standard_logging_payload(
@@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
             time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
         )
         end_time_ns: Final = to_ns(end_time)
-        if carrier.span is not None:
+        if carrier is not None and carrier.span is not None:
             # Born at the boundary: stamp attributes from the typed payload, set
             # status, and end it. Its parent (the server span) was captured at
-            # creation from real ambient context.
+            # creation from real ambient context. Register the dedup marker so a
+            # second close for the same call id (success then failure on one
+            # logging object) cannot re-emit through the deferred branch.
+            self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
             self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
             return carrier.span
         # Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
                 SpanRole.LLM_CALL,
                 data,
                 parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
-                start_time_ns=carrier.start_time_ns,
+                start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
                 end_time_ns=end_time_ns,
                 tracer=route.tracer,
                 links=_request_trace_links(parent_ctx) if route.detached else None,

litellm/integrations/otel/model/config.py

@@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
     WEAVE_OTEL = "weave_otel"
     LEVO = "levo"
     AGENTOPS = "agentops"
+    NEWRELIC = "newrelic"
 
 
 class _OTelV2Flag(BaseSettings):
@@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
             "auto (Simple for console/in_memory, Batch otherwise)."
         ),
     )
+    requires_headers: bool = Field(
+        default=False,
+        description=(
+            "Skip this exporter when no headers are resolved. For destinations "
+            "that reject unauthenticated exports (e.g. New Relic), a spec kept "
+            "only as the per-request credential-stamping target would otherwise "
+            "export keyless traffic and produce a 4xx for every span batch."
+        ),
+    )
 
 
 class OpenTelemetryV2Config(BaseSettings):

litellm/integrations/otel/model/metadata.py

@@ -203,6 +203,11 @@ class LLMCallEvent:
     # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
     # the ``pre_call`` hook but never made an upstream call, so they get no span.
     is_no_upstream_call: bool
+    # True once the request handed off to a provider (``pre_call`` stamped
+    # ``api_call_start_time``). The affirmative signal that an LLM call was
+    # actually attempted — router pre-call rejections, SDK failures before the
+    # provider handoff, and standalone guardrail runs all lack it.
+    upstream_started: bool
     # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
     # span is renamed from the typed payload at close (``finish_span``); this only
     # needs to be reasonable for a span that never gets closed (a leak).
@@ -221,6 +226,7 @@ class LLMCallEvent:
             dynamic_params=kwargs.get("standard_callback_dynamic_params"),
             auth_metadata=auth_metadata(payload, kwargs),
             is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
+            upstream_started=kwargs.get("api_call_start_time") is not None,
             provisional_span_name=f"{operation.value} {model}".strip(),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
         )

litellm/integrations/otel/plumbing/providers.py

@@ -436,6 +436,8 @@ def build_tracer_provider(
     # ``config._normalize`` guarantees at least one spec (it folds the top-level
     # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
     for spec in config.exporters:
+        if spec.requires_headers and not spec.headers:
+            continue
         exp = _exporter_from_spec(spec)
         provider.add_span_processor(
             _processor_for(

litellm/integrations/otel/plumbing/routing.py

@@ -28,6 +28,7 @@ from litellm.integrations.otel.plumbing.providers import (
     get_tracer,
 )
 from litellm.integrations.otel.presets import (
+    dynamic_otlp_endpoint,
     dynamic_otlp_headers,
     project_routing_headers,
 )
@@ -129,7 +130,9 @@ class TenantTracerCache:
         # thread-pool workers concurrently with the event loop, so cache
         # updates, span counts, and retirement must be atomic.
         self._lock: Final = threading.Lock()
-        self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
+        self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems, str | None], TracerProvider] = (
+            OrderedDict()  # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
+        )
         self._open_span_counts: dict[TracerProvider, int] = {}  # mutable-ok: live refcount state
         # Oldest-first so an overflow of draining providers sheds the stalest.
         self._retired: OrderedDict[TracerProvider, None] = OrderedDict()  # mutable-ok: draining evicted providers
@@ -182,12 +185,16 @@ class TenantTracerCache:
         project_headers: Final = self._project_headers(auth_metadata)
         if not credential_headers and not project_headers:
             return TenantRoute(tracer=default, detached=False)
+        # A fixed per-integration region endpoint (New Relic us/eu), never a
+        # caller-supplied host; ``None`` keeps the preset's own endpoint.
+        endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
         cache_key: Final = (
             tuple(sorted(credential_headers.items())),
             tuple(sorted(project_headers.items())),
+            endpoint,
         )
         with self._lock:
-            provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
+            provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers, endpoint)
             self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
             evicted: Final = self._evicted_on_overflow_locked()
         if evicted is not None:
@@ -200,15 +207,16 @@ class TenantTracerCache:
 
     def _cached_provider_locked(
         self,
-        cache_key: tuple[_HeaderItems, _HeaderItems],
+        cache_key: tuple[_HeaderItems, _HeaderItems, str | None],
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None,
     ) -> TracerProvider:
         cached: Final = self._providers.get(cache_key)
         if cached is not None:
             self._providers.move_to_end(cache_key)
             return cached
-        built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
+        built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers, endpoint))
         self._providers[cache_key] = built
         return built
 
@@ -257,6 +265,7 @@ class TenantTracerCache:
         self,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None = None,
     ) -> OpenTelemetryV2Config:
         """Clone the config, rewriting headers on the callback's own exporter.
 
@@ -272,7 +281,8 @@ class TenantTracerCache:
         ``Authorization``), which must survive routing to a project.
         """
         exporters: Final = [
-            self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
+            self._routed_exporter(spec, credential_headers, project_headers, endpoint)
+            for spec in self._config.exporters
         ]
         return self._config.model_copy(update={"exporters": exporters})
 
@@ -281,6 +291,7 @@ class TenantTracerCache:
         spec: ExporterSpec,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None = None,
     ) -> ExporterSpec:
         kind: Final = spec.kind.lower()
         if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@@ -291,4 +302,10 @@ class TenantTracerCache:
             if project_headers and kind not in _GRPC_KINDS
             else base
         )
-        return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
+        update: Final = {  # mutable-ok: model_copy(update=...) requires a plain dict
+            field: value
+            for field, value in (("headers", routed), ("endpoint", endpoint))
+            if (field == "headers" and routed != spec.headers)
+            or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
+        }
+        return spec if not update else spec.model_copy(update=update)

litellm/integrations/otel/presets/__init__.py

@@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
 )
 from litellm.integrations.otel.presets.langtrace import langtrace_preset
 from litellm.integrations.otel.presets.levo import levo_preset
+from litellm.integrations.otel.presets.newrelic import (
+    newrelic_dynamic_endpoint,
+    newrelic_dynamic_headers,
+    newrelic_preset,
+)
 from litellm.integrations.otel.presets.phoenix import (
     phoenix_preset,
     phoenix_project_headers,
@@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
 
 #: Callback name → preset. The ``Preset`` annotation makes mypy verify every
 #: registered value matches the preset interface.
-PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
-    "agentops": agentops_preset,
-    "arize": arize_preset,
-    "arize_phoenix": phoenix_preset,
-    "langfuse_otel": langfuse_preset,
-    "langtrace": langtrace_preset,
-    "levo": levo_preset,
-    "weave_otel": weave_preset,
-}
+PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
+    {
+        "agentops": agentops_preset,
+        "arize": arize_preset,
+        "arize_phoenix": phoenix_preset,
+        "langfuse_otel": langfuse_preset,
+        "langtrace": langtrace_preset,
+        "levo": levo_preset,
+        "newrelic": newrelic_preset,
+        "weave_otel": weave_preset,
+    }
+)
 
 #: Callback name → per-request OTLP header builder (team/key multi-tenant
 #: routing). Only integrations that support dynamic credentials appear here —
 #: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
 #: default tracer.
-DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
-    "arize": arize_dynamic_headers,
-    "langfuse_otel": langfuse_dynamic_headers,
-    "weave_otel": weave_dynamic_headers,
-}
+DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
+    MappingProxyType(
+        {
+            "arize": arize_dynamic_headers,
+            "langfuse_otel": langfuse_dynamic_headers,
+            "newrelic": newrelic_dynamic_headers,
+            "weave_otel": weave_dynamic_headers,
+        }
+    )
+)
+
+#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
+#: destination host varies per tenant (from a fixed region table, never a
+#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
+#: authoritative.
+DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
+    MappingProxyType(
+        {
+            "newrelic": newrelic_dynamic_endpoint,
+        }
+    )
+)
 
 
 #: Callback name → per-request *routing* header builder, sourced from the key/team
@@ -98,17 +123,34 @@ def project_routing_headers(
     return builder(auth_metadata)
 
 
+def dynamic_otlp_endpoint(
+    callback_name: str | None,
+    dynamic_params: StandardCallbackDynamicParams | None,
+) -> str | None:
+    """Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
+
+    ``None`` means "keep the preset's own endpoint".
+    """
+    resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
+    if resolver is None or not dynamic_params:
+        return None
+    return resolver(dynamic_params)
+
+
 __all__ = [
+    "DYNAMIC_ENDPOINT_BY_CALLBACK",
     "DYNAMIC_HEADERS_BY_CALLBACK",
     "PRESET_BY_CALLBACK",
     "PROJECT_HEADERS_BY_CALLBACK",
     "Preset",
     "agentops_preset",
     "arize_preset",
+    "dynamic_otlp_endpoint",
     "dynamic_otlp_headers",
     "langfuse_preset",
     "langtrace_preset",
     "levo_preset",
+    "newrelic_preset",
     "phoenix_preset",
     "project_routing_headers",
     "weave_preset",

litellm/integrations/otel/presets/newrelic.py

@@ -0,0 +1,104 @@
+"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from litellm._logging import verbose_logger
+from litellm.integrations.otel.model.config import (
+    ExporterOwner,
+    ExporterSpec,
+    OpenTelemetryV2Config,
+)
+from litellm.integrations.otel.presets.utils import ensure_mappers
+from litellm.types.utils import StandardCallbackDynamicParams
+
+#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
+#: region enum rather than a free-form endpoint, so callback vars can never
+#: redirect telemetry to an arbitrary host.
+NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "us": "https://otlp.nr-data.net",
+        "eu": "https://otlp.eu01.nr-data.net",
+    }
+)
+
+_DEFAULT_REGION: Final = "us"
+
+
+class _NewRelicSettings(BaseSettings):
+    model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
+
+    # The same env vars the agent-based integration documents; the key is the
+    # operator-level fallback for traffic without team credentials, the region
+    # picks that fallback's data center, and the record-content flag keeps its
+    # documented meaning when the OTel path replaces the agent.
+    license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
+    region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
+    record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
+
+
+def newrelic_preset(
+    *,
+    config_overrides: OpenTelemetryV2Config | None = None,
+) -> OpenTelemetryV2Config:
+    settings: Final = _NewRelicSettings()
+    base: Final = config_overrides or OpenTelemetryV2Config()
+    endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
+        (settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
+    )
+    return base.model_copy(
+        update={
+            "exporters": [
+                *base.exporters,
+                ExporterSpec(
+                    kind="otlp_http",
+                    endpoint=endpoint,
+                    headers=(f"api-key={settings.license_key}" if settings.license_key else None),
+                    owner=ExporterOwner.NEWRELIC,
+                    requires_headers=True,
+                ),
+            ],
+            # New Relic ingests the OTLP GenAI semantic conventions natively.
+            "mapper_names": ensure_mappers(base.mapper_names, "genai"),
+            **(
+                {"capture_message_content": ("span_only" if settings.record_content else "no_content")}
+                if settings.record_content is not None
+                else {}
+            ),
+        }
+    )
+
+
+def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
+    """Per-request New Relic OTLP headers from team/key dynamic params."""
+    api_key: Final = params.get("newrelic_api_key")
+    return {header: value for header, value in (("api-key", api_key),) if value}
+
+
+def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
+    """Per-request OTLP endpoint for the team's ``newrelic_region``.
+
+    Always the team's own region endpoint, defaulting to US when the team left
+    the region unset. It never falls through to the preset's endpoint, which
+    follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
+    ingest key must not inherit the operator's region and have its US-account
+    spans rejected by an EU-configured default (or vice versa). An unknown
+    region likewise resolves to the documented US default rather than a guess.
+    """
+    region: Final = params.get("newrelic_region")
+    default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
+    if not region:
+        return default_endpoint
+    endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
+    if endpoint is None:
+        verbose_logger.warning(
+            "New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
+            region,
+            ", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
+        )
+        return default_endpoint
+    return endpoint

litellm/integrations/prometheus.py

@@ -96,7 +96,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
 
 def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
     """View a repository's prisma table through the pagination surface budget metrics need."""
-    return repository.table
+    return cast(
+        _PaginatedPrismaTable[_TableRowT],
+        repository.table,  # cast-ok: prisma rows carry the budget columns the domain model declares
+    )
 
 
 class _OrgBudgetRow(Protocol):

litellm/integrations/s3_v2.py

@@ -11,6 +11,7 @@ import time
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Final, cast
+from urllib.parse import quote
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
         )
 
+    def _build_object_url(self, s3_object_key: str) -> str:
+        """
+        Build the exact URL that is both signed and sent, with the key percent-encoded once.
+
+        S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
+        characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
+        the two signatures disagree (403 SignatureDoesNotMatch).
+        """
+        encoded_key: Final = quote(s3_object_key, safe="/")
+        if self.s3_endpoint_url and self.s3_bucket_name:
+            if self.s3_use_virtual_hosted_style:
+                endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
+                protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
+                return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
+            return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
+        return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
+
     def _sse_headers(self) -> Mapping[str, str]:
         candidates: Final = {
             "x-amz-server-side-encryption": self.s3_server_side_encryption,
@@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
             verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
             from botocore.credentials import Credentials
@@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 aws_region_name=self.s3_region_name,
             )
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             httpx_client: Final = _get_httpx_client(
                 params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
             )
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         try:
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
 
             verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
+            url: Final = self._build_object_url(s3_object_key)
 
             # Prepare the request for GET operation
             # For GET requests, we need x-amz-content-sha256 with hash of empty string
@@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             headers: Final = {
                 "x-amz-content-sha256": empty_string_hash,
             }
-            req: Final = requests.Request("GET", url, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
             S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            request_url: Final = prepped.url or url
-            response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
+            response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
 
             if response.status_code != 200:
                 verbose_logger.exception("S3 object not found, saw response=", response.text)

litellm/interactions/background_cost_polling.py

@@ -0,0 +1,313 @@
+"""
+Cost tracking for background interactions.
+
+A create request with ``background=true`` returns ``in_progress`` with no
+usage block, and GET polls are deliberately never billed (billing them would
+double-charge every poll; the GET response also does not echo ``background``,
+so a poll cannot be told apart from a re-fetch of an already-billed
+interaction). The create call is therefore the only place that can own
+billing: it schedules a poll task that fetches the interaction until it
+reaches a terminal status and logs the final usage as a single success event
+attributed to the original request.
+
+``requires_action`` is terminal for the interaction it names. The API has no
+operation that resumes one: a caller answers a tool request by creating a new
+interaction whose ``previous_interaction_id`` points at it, and that new
+interaction bills itself. The paused interaction keeps the tokens it already
+spent producing the tool request, so it is billed and settled where it stops
+rather than polled until the timeout, which would both lose that usage and
+hold its budget reservation open for the whole timeout window.
+
+Deleting an interaction makes every subsequent poll fail, which would let a
+caller retrieve the completed output themselves and then delete it before the
+poll task settles, leaving the work unbilled and the budget reservation
+refunded at the poll timeout. ``adelete`` therefore settles any pending poll
+for the interaction before dispatching the delete: it fetches the current
+state with the create's credentials, bills it if it is terminal with usage,
+and releases the reservation otherwise. A settlement gate on the create's
+logging object makes the poll task and the delete path mutually exclusive, so
+the interaction is billed exactly once no matter who settles first.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
+)
+from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
+from litellm.types.interactions import InteractionsAPIResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_TERMINAL_STATUSES: Final = frozenset(
+    {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
+)
+
+_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
+
+_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
+
+
+@dataclass(frozen=True, slots=True)
+class BackgroundInteractionPollContext:
+    interaction_id: str
+    custom_llm_provider: str
+    logging_obj: "LiteLLMLoggingObj"
+    api_key: str | None = None
+    api_base: str | None = None
+    initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
+    max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
+    timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
+
+
+FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
+
+
+async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
+    from litellm.interactions import aget
+
+    return await aget(
+        interaction_id=context.interaction_id,
+        custom_llm_provider=context.custom_llm_provider,
+        api_key=context.api_key,
+        api_base=context.api_base,
+        **{
+            "no-log": True
+        },  # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
+    )
+
+
+def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
+    elapsed = 0.0
+    interval = initial
+    while interval > 0 and elapsed + interval <= timeout:
+        yield interval
+        elapsed += interval
+        interval = min(interval * 2, maximum)
+
+
+_SETTLED_KEY = "background_interaction_settled"
+
+
+def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
+    return logging_obj.model_call_details.get(_SETTLED_KEY) is True
+
+
+def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
+    """
+    Exactly-once gate between the poll task and the delete-time settlement:
+    both run on the same event loop and neither awaits between reading and
+    setting the flag, so whichever claims first owns billing or release.
+    """
+    if _is_settled(logging_obj):
+        return False
+    logging_obj.model_call_details[_SETTLED_KEY] = True  # rebind-ok: both settlers must see the same settlement flag
+    return True
+
+
+async def poll_and_log_background_interaction_cost(
+    context: BackgroundInteractionPollContext,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    last_seen_status: str | None = None
+    for interval in _poll_intervals(
+        initial=context.initial_interval_seconds,
+        maximum=context.max_interval_seconds,
+        timeout=context.timeout_seconds,
+    ):
+        await asyncio.sleep(interval)
+        if _is_settled(context.logging_obj):
+            return
+        try:
+            response = await fetch_interaction(context)
+        except Exception as e:  # noqa: BLE001  # any fetch error must not kill the billing poll loop
+            verbose_logger.debug(
+                "Background interaction cost poll for %s failed, will retry: %s",
+                context.interaction_id,
+                e,
+            )
+            continue
+        last_seen_status = response.status
+        if response.status not in _TERMINAL_STATUSES:
+            continue
+        if not _claim_settlement(context.logging_obj):
+            return
+        if response.usage is not None:
+            await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        else:
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
+        verbose_logger.error(
+            "Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
+            "the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
+            "will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+            last_seen_status,
+        )
+    else:
+        verbose_logger.warning(
+            "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+        )
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)
+
+
+async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
+    """
+    The proxy keeps the pre-call budget reservation open for an in-progress
+    background interaction so concurrent creates cannot stack past the budget.
+    The completion success event reconciles it to the actual cost; when the
+    interaction terminates without billable usage (or polling gives up, or it
+    is deleted before settling), no such event fires, so whoever claims the
+    settlement must release the reservation here or the spend counters stay
+    pinned at the estimated cost.
+    """
+    metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
+    budget_reservation = metadata.get("user_api_key_budget_reservation")
+    if not isinstance(budget_reservation, dict):
+        return
+
+    from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
+
+    try:
+        await release_budget_reservation(budget_reservation=budget_reservation)
+    except Exception:  # noqa: BLE001  # a failed release must not crash the poll task; counters expire via TTL
+        verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
+
+
+async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
+    """
+    Claiming the settlement makes the claimer solely responsible for the
+    reservation, and no one retries a claim that is already set. A billing
+    failure here must therefore release the reservation on its way out, or it
+    stays pinned at the estimated cost until the whole poll times out.
+    """
+    try:
+        await logging_obj.async_log_background_interaction_completion(result=response)
+    except Exception:
+        await _release_open_budget_reservation(logging_obj=logging_obj)
+        raise
+
+
+def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
+    """
+    The single gate deciding whether a create's response gets a poll task.
+    The proxy's success callback defers releasing the budget reservation for
+    exactly these responses, on the promise that a poll task will settle them,
+    so a response one site accepts and the other refuses strands its
+    reservation on the spend counters with nothing left to reconcile it.
+
+    ``queued`` belongs here alongside ``in_progress``. It is the API's
+    not-started-yet state, so it reaches a terminal status the same way and
+    needs polling for the same reason: nothing else in the proxy ever bills a
+    create that came back without usage, so a status missing from both this
+    set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
+    """
+    return response.status in _POLLABLE_STATUSES and bool(response.id)
+
+
+def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
+    """
+    Whether a response arriving with no usage block is a normal outcome rather
+    than lost billing data. An interaction that is still running, or that
+    stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
+    has nothing to charge for and should not raise a cost-tracking alarm.
+
+    ``completed`` and ``requires_action`` both mean the model produced output,
+    so a usage block is always expected with them. If one arrives without it
+    the charge for real work has been lost, which is precisely what the
+    proxy's cost-tracking alert exists to surface.
+    """
+    return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
+
+
+@dataclass(frozen=True, slots=True)
+class _ActiveBackgroundPoll:
+    task: "asyncio.Task[None]"
+    context: BackgroundInteractionPollContext
+
+
+_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {}  # mutable-ok: asyncio needs strong refs to running poll tasks
+
+
+def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is not None and entry.task is task:
+        del _ACTIVE_POLLS[interaction_id]
+
+
+def maybe_schedule_background_interaction_cost_polling(
+    response: object,
+    create_kwargs: Mapping[str, object],
+    custom_llm_provider: str,
+) -> "asyncio.Task[None] | None":
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
+        return None
+    if not isinstance(response, InteractionsAPIResponse):
+        return None
+    if not is_pollable_background_interaction(response):
+        return None
+    logging_obj = create_kwargs.get("litellm_logging_obj")
+    if not isinstance(logging_obj, Logging):
+        return None
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+    api_key = create_kwargs.get("api_key")
+    api_base = create_kwargs.get("api_base")
+    context = BackgroundInteractionPollContext(
+        interaction_id=response.id,
+        custom_llm_provider=custom_llm_provider,
+        logging_obj=logging_obj,
+        api_key=api_key if isinstance(api_key, str) else None,
+        api_base=api_base if isinstance(api_base, str) else None,
+    )
+    task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
+    _ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
+    task.add_done_callback(
+        lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
+    )
+    return task
+
+
+async def maybe_settle_background_interaction_before_delete(
+    interaction_id: str,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is None:
+        return
+    context = entry.context
+    try:
+        response = await fetch_interaction(context)
+    except Exception as e:  # noqa: BLE001  # unfetchable pre-delete state settles by releasing the reservation
+        verbose_logger.debug(
+            "Could not fetch background interaction %s before delete, releasing its reservation: %s",
+            interaction_id,
+            e,
+        )
+        if _claim_settlement(context.logging_obj):
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if response.status in _TERMINAL_STATUSES and response.usage is not None:
+        await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        return
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)

litellm/interactions/main.py

@@ -40,6 +40,10 @@ from typing import Any, Final
 import httpx
 
 import litellm
+from litellm.interactions.background_cost_polling import (
+    maybe_schedule_background_interaction_cost_polling,
+    maybe_settle_background_interaction_before_delete,
+)
 from litellm.interactions.http_handler import interactions_http_handler
 from litellm.interactions.utils import (
     InteractionsAPIRequestUtils,
@@ -171,6 +175,12 @@ async def acreate(
         else:
             response = init_response
 
+        maybe_schedule_background_interaction_cost_polling(
+            response=response,
+            create_kwargs=kwargs,
+            custom_llm_provider=custom_llm_provider,
+        )
+
         return response
     except Exception as e:
         raise litellm.exception_type(
@@ -462,6 +472,8 @@ async def adelete(
         loop: Final = asyncio.get_event_loop()
         kwargs["adelete_interaction"] = True
 
+        await maybe_settle_background_interaction_before_delete(interaction_id=interaction_id)
+
         func: Final = partial(
             delete,
             interaction_id=interaction_id,

litellm/interactions/utils.py

@@ -47,6 +47,13 @@ def get_provider_interactions_api_config(
 
         return GoogleAIStudioInteractionsConfig()
 
+    if provider in (LlmProviders.VERTEX_AI.value, LlmProviders.VERTEX_AI_BETA.value):
+        from litellm.llms.vertex_ai.interactions.transformation import (
+            VertexAIInteractionsConfig,
+        )
+
+        return VertexAIInteractionsConfig()
+
     return None
 
 

litellm/litellm_core_utils/core_helpers.py

@@ -58,6 +58,67 @@ def safe_divide(
     return numerator / denominator
 
 
+def _is_litellm_limit_rejection(exception: BaseException) -> bool:
+    from litellm.exceptions import RateLimitErrorCategory
+
+    litellm_limit_categories: Final = frozenset(
+        (RateLimitErrorCategory.LITELLM_RATE_LIMIT.value, RateLimitErrorCategory.LITELLM_BATCH_RATE_LIMIT.value)
+    )
+    return getattr(exception, "category", None) in litellm_limit_categories
+
+
+def _is_proxy_rejection(exception: BaseException) -> bool:
+    if _is_litellm_limit_rejection(exception):
+        return True
+    try:
+        from starlette.exceptions import HTTPException
+    except ImportError:
+        return False
+    return isinstance(exception, HTTPException)
+
+
+def _is_provider_originated(exception: BaseException) -> bool:
+    if _is_proxy_rejection(exception):
+        return False
+    if getattr(exception, "llm_provider", None):
+        return True
+    from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+    return isinstance(exception, BaseLLMException)
+
+
+def is_expected_client_error(exception: BaseException | None) -> bool:
+    """
+    True when the proxy itself rejected the request with an HTTP 4xx before any
+    provider call (bad key, budget, unknown model, guardrail). A 4xx returned by
+    a provider is an upstream or deployment problem, so it is never an expected
+    client error and keeps its traceback: a mapped litellm exception carries
+    ``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
+    raise before mapping (the /v1/messages route surfaces it as-is) is one too.
+    The proxy's own limiters raise ``HTTPException`` subclasses that also carry
+    an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection, and
+    so does any exception whose unified rate-limit ``category`` names litellm's
+    own limiter (``BudgetExceededError`` is a plain ``Exception`` that the auth
+    handler decorates with the requested model's provider).
+
+    ProxyException stores the status on .code (as a str), HTTPException and
+    litellm exceptions on .status_code.
+    """
+    if exception is None:
+        return False
+    if _is_provider_originated(exception):
+        return False
+    code: Final[object] = getattr(exception, "code", None)
+    status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
+    if status_code is None or isinstance(status_code, bool):
+        return False
+    try:
+        status: Final = int(str(status_code))
+    except ValueError:
+        return False
+    return 400 <= status < 500
+
+
 def coerce_token_limit(value: object) -> int | None:
     """
     Coerce a max_input_tokens / max_output_tokens value to an int, treating a

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2301,6 +2301,7 @@ def exception_type(
                 or custom_llm_provider == "custom_openai"
                 or custom_llm_provider in litellm.openai_compatible_providers
                 or custom_llm_provider == "mistral"
+                or custom_llm_provider == "runwayml"
             ):
                 _map_openai_exception(
                     model=model,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -272,6 +272,14 @@ def get_llm_provider(
                     elif endpoint == "api.deepseek.com/v1":
                         custom_llm_provider = "deepseek"
                         dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
+                    elif endpoint == "api.together.ai/v1" or endpoint == "api.together.xyz/v1":
+                        custom_llm_provider = "together_ai"
+                        dynamic_api_key = api_key or (
+                            get_secret_str("TOGETHER_API_KEY")
+                            or get_secret_str("TOGETHER_AI_API_KEY")
+                            or get_secret_str("TOGETHERAI_API_KEY")
+                            or get_secret_str("TOGETHER_AI_TOKEN")
+                        )
                     elif endpoint == "ollama.com":
                         custom_llm_provider = "ollama"
                         dynamic_api_key = get_secret_str("OLLAMA_API_KEY")
@@ -707,7 +715,7 @@ def _get_openai_compatible_provider_info(
             dynamic_api_key,
         ) = litellm.ZAIChatConfig()._get_openai_compatible_provider_info(api_base, api_key)
     elif custom_llm_provider == "together_ai":
-        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.xyz/v1"
+        api_base = api_base or get_secret_str("TOGETHER_AI_API_BASE") or "https://api.together.ai/v1"
         dynamic_api_key = api_key or (
             get_secret_str("TOGETHER_API_KEY")
             or get_secret_str("TOGETHER_AI_API_KEY")

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -172,7 +172,7 @@ def get_supported_openai_params(
         if request_type == "embeddings":
             return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "together_ai":
-        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
+        return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "databricks":
         if request_type == "chat_completion":
             return litellm.DatabricksConfig().get_supported_openai_params(model=model)

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -46,7 +46,7 @@ def validate_no_callback_env_reference(param: str, value: object, *, source: str
 
 
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
-_supported_callback_params: Final = [
+_supported_callback_params: Final[tuple[str, ...]] = (
     "langfuse_public_key",
     "langfuse_secret",
     "langfuse_secret_key",
@@ -72,8 +72,10 @@ _supported_callback_params: Final = [
     "dd_site",
     "dd_agent_host",
     "dd_agent_port",
+    "newrelic_api_key",
+    "newrelic_region",
     "turn_off_message_logging",
-]
+)
 
 _request_blocked_callback_params: Final = frozenset(
     {
@@ -83,6 +85,20 @@ _request_blocked_callback_params: Final = frozenset(
         "dd_site",
         "dd_agent_host",
         "dd_agent_port",
+        "newrelic_api_key",
+        "newrelic_region",
+    }
+)
+
+# Request-blocked params that must still reach ``standard_callback_dynamic_params``
+# when the proxy itself stamped them from admin-configured team/key callback
+# settings (the trusted-vars channel). The OTel per-tenant tracer routing reads
+# ``standard_callback_dynamic_params``, so without this overlay a blocked param
+# could never drive routing at all.
+_trusted_overlay_callback_params: Final = frozenset(
+    {
+        "newrelic_api_key",
+        "newrelic_region",
     }
 )
 
@@ -121,7 +137,9 @@ def initialize_standard_callback_dynamic_params(
             if param in kwargs:
                 _param_value = kwargs.get(param)
                 validate_no_callback_env_reference(param, _param_value, source="request body")
-                standard_callback_dynamic_params[param] = _param_value
+                standard_callback_dynamic_params[param] = (  # pyright: ignore[reportGeneralTypeIssues]  # several supported params predate their StandardCallbackDynamicParams fields
+                    _param_value
+                )
 
         for slot_label, metadata in iter_client_callback_metadata_dicts(kwargs):
             for param in _supported_callback_params:
@@ -130,6 +148,12 @@ def initialize_standard_callback_dynamic_params(
                 if param not in standard_callback_dynamic_params and param in metadata:
                     _param_value = metadata.get(param)
                     validate_no_callback_env_reference(param, _param_value, source=slot_label)
-                    standard_callback_dynamic_params[param] = _param_value
+                    standard_callback_dynamic_params[param] = (  # pyright: ignore[reportGeneralTypeIssues]  # several supported params predate their StandardCallbackDynamicParams fields
+                        _param_value
+                    )
+
+        for param, trusted_value in get_trusted_callback_params(kwargs):
+            if param in _trusted_overlay_callback_params:
+                standard_callback_dynamic_params[param] = trusted_value
 
     return standard_callback_dynamic_params

litellm/litellm_core_utils/json_fragment_accumulator.py

@@ -0,0 +1,97 @@
+import json
+from typing import Final, cast  # noqa: TID251  # raw_decode returns tuple[Any, int]; no cast-free unpack
+
+
+class JSONFragmentAccumulator:
+    """
+    Buffers a JSON value that arrives piecemeal over a stream (SSE data split
+    across TCP packets, one shard per network read, etc) without the O(n^2)
+    cost of repeated `buffer += fragment` string concatenation, and without
+    the O(n^2) cost of re-copying the unconsumed remainder on every peeled
+    value when one payload holds many concatenated JSON values.
+
+    Fragments are appended to a list in O(1). The buffer is only rebuilt into
+    a single string, and only decoded, when a caller asks for a value via
+    `pop_next_value`, and `could_close_json` lets callers skip that rebuild
+    entirely for fragments that plainly cannot close a JSON value yet. Once
+    rebuilt, consumed values are dropped by advancing a cursor rather than
+    slicing a new string, so draining N concatenated values already sitting
+    in the buffer costs O(n) total, not O(n^2).
+    """
+
+    def __init__(self) -> None:
+        self._chunks: list[str] = []  # mutable-ok: O(1) append; string concat would copy the buffer each time
+        self._buffer: str = (
+            ""  # mutable-ok: lazily materialized join of _chunks, rebuilt only when _chunks is non-empty
+        )
+        self._offset: int = 0  # mutable-ok: cursor past already-consumed values; avoids re-slicing on every pop
+        self._could_close: bool = False  # mutable-ok: cached heuristic; rescanning past fragments was itself O(n^2)
+
+    def __bool__(self) -> bool:
+        return bool(self._chunks) or self._offset < len(self._buffer)
+
+    def append(self, fragment: str) -> None:
+        self._chunks.append(fragment)  # mutable-ok: see __init__
+        stripped: Final = fragment.rstrip()
+        if stripped:
+            self._could_close = stripped[-1] in ("}", "]")  # mutable-ok: see __init__
+
+    def could_close_json(self) -> bool:
+        """
+        Whether the buffer's logical last non-whitespace byte is "}" or "]",
+        i.e. whether a JSON value could plausibly be complete. Tracked
+        incrementally in `append` rather than rescanned here, so a run of
+        blank keepalive fragments (e.g. from a malformed upstream stream)
+        can't make this, or the join+parse it gates, cost O(n^2).
+        """
+        return self._could_close
+
+    def _materialize(self) -> None:
+        if not self._chunks:
+            return
+        unconsumed: Final = self._buffer[self._offset :]
+        self._buffer = unconsumed + "".join(self._chunks)  # mutable-ok: merge pending fragments, once per append batch
+        self._offset = 0  # mutable-ok: see __init__
+        self._chunks = []  # mutable-ok: see __init__
+
+    def pop_next_value(self) -> tuple[bool, object]:
+        """
+        Attempt to decode one complete JSON value from the front of the
+        buffer. On success, advances a cursor past that value (keeping any
+        unconsumed tail, e.g. a second concatenated value, in place rather
+        than copying it) and returns (True, value). If the buffer is empty
+        or holds no complete value yet, it is left untouched and this
+        returns (False, None).
+        """
+        self._materialize()
+        length: Final = len(self._buffer)
+        start = self._offset
+        while start < length and self._buffer[start].isspace():
+            start += 1
+        if start >= length:
+            self._offset = start  # mutable-ok: see __init__
+            return False, None
+        decoder: Final = json.JSONDecoder()
+        try:
+            raw_value: Final = decoder.raw_decode(self._buffer, start)
+        except json.JSONDecodeError:
+            return False, None
+        decoded, end_index = cast("tuple[object, int]", raw_value)  # cast-ok: raw_decode returns tuple[Any, int]
+        self._offset = end_index  # mutable-ok: see __init__
+        if self._offset >= len(self._buffer):
+            self._buffer = ""  # mutable-ok: see __init__
+            self._offset = 0  # mutable-ok: see __init__
+            self._could_close = False  # mutable-ok: buffer is empty, nothing can close
+        return True, decoded
+
+    def snapshot(self) -> str:
+        self._materialize()
+        return self._buffer[self._offset :]
+
+    def set(self, value: str) -> None:
+        """Replace the buffer's contents with a single fragment."""
+        self._chunks = []  # mutable-ok: see __init__
+        self._buffer = value  # mutable-ok: see __init__
+        self._offset = 0  # mutable-ok: see __init__
+        stripped: Final = value.rstrip()
+        self._could_close = bool(stripped) and stripped[-1] in ("}", "]")  # mutable-ok: see __init__

litellm/litellm_core_utils/litellm_logging.py

@@ -62,7 +62,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
-from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
+from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
@@ -71,6 +71,9 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
+from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
+)
 from litellm.litellm_core_utils.logging_utils import truncate_base64_in_messages
 from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
 from litellm.litellm_core_utils.redact_messages import (
@@ -83,6 +86,10 @@ from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.responses.utils import ResponseAPILoggingUtils
 from litellm.types.agents import LiteLLMSendMessageResponse
 from litellm.types.containers.main import ContainerObject
+from litellm.types.interactions import (
+    InteractionsAPIResponse,
+    InteractionsAPIStreamingResponse,
+)
 from litellm.types.llms.openai import (
     AllMessageValues,
     Batch,
@@ -2145,6 +2152,11 @@ class Logging(LiteLLMLoggingBaseClass):
             or isinstance(logging_result, OpenAIModerationResponse)
             or isinstance(logging_result, OCRResponse)  # OCR
             or isinstance(logging_result, SearchResponse)  # Search API
+            or (
+                isinstance(logging_result, InteractionsAPIResponse)
+                and logging_result.usage is not None
+                and self._is_interactions_create_call_type()
+            )
             or isinstance(logging_result, dict)
             and logging_result.get("object") == "vector_store.search_results.page"
             or isinstance(logging_result, dict)
@@ -2157,6 +2169,87 @@ class Logging(LiteLLMLoggingBaseClass):
             return True
         return False
 
+    def _is_interactions_create_call_type(self) -> bool:
+        """
+        Only interaction creation is billable. GET polls, deletes, and cancels
+        also return an ``InteractionsAPIResponse`` (with usage once completed),
+        so recognizing those would write spend on every poll of a background
+        interaction. The proxy sets ``call_type`` from its route_type
+        (``create_interaction``/``acreate_interaction``); the SDK sets it from
+        the decorated function name (``create``/``acreate``).
+
+        Recognition additionally requires a usage block (checked at the call
+        site): a ``background=true`` create returns ``in_progress`` without
+        usage, and billing it would write a $0 spend log under the interaction
+        id that collides with the row the background poll task writes once the
+        interaction completes (see
+        ``litellm.interactions.background_cost_polling``).
+        """
+        return self.call_type in (
+            CallTypes.create_interaction.value,
+            CallTypes.acreate_interaction.value,
+            "create",
+            "acreate",
+        )
+
+    async def async_log_background_interaction_completion(
+        self,
+        result: InteractionsAPIResponse,
+    ) -> None:
+        """
+        Log the terminal result of a background interaction as a fresh success
+        event. The create request already ran success logging for its
+        ``in_progress`` response (no usage, so no cost was tracked); clearing
+        the dedup flags lets the completed result flow through cost calculation
+        and spend tracking exactly once, spanning create to completion.
+
+        The poll fetched this body through its own client call, which priced it
+        against a throwaway logging object holding none of this request's
+        deployment context: no ``model_info``, no router ``model_id``, no
+        deployment ``litellm_params``. Keeping that price would bill a
+        custom-priced deployment at the wrong rate, and it would also satisfy
+        the "already calculated" shortcut and skip repricing here, leaving the
+        cost breakdown at the zeros the usage-less create stamped and writing
+        those zeros to the spend log. Dropping it makes this event price the
+        settled body itself, against the deployment that served the create.
+
+        The same throwaway call stamped the deployment identity that travels
+        with the price, so ``model_id`` and ``litellm_model_name`` go with it.
+        Left in place they overwrite the create's real deployment with the
+        poll's empty one in the payload every logging integration reads.
+        """
+        settled_hidden_params: Final = getattr(result, "_hidden_params", None)
+        if isinstance(settled_hidden_params, dict):
+            for poll_scoped_key in ("response_cost", "model_id", "litellm_model_name"):
+                settled_hidden_params.pop(poll_scoped_key, None)
+        self._reset_success_emission_dedupe()
+        await self.async_success_handler(result=result)
+
+    def _reset_success_emission_dedupe(self) -> None:
+        """
+        Success callbacks dedupe per request, because the sync and async
+        handlers both fire on some paths and would otherwise report one call
+        twice. A settled background interaction is a genuinely second success
+        event on the same request, so every such marker has to be cleared or
+        the completion, the only event that carries usage and cost, is
+        discarded as a duplicate of the in-progress create.
+        """
+        self.model_call_details.pop("has_logged_async_success", None)
+        litellm_params = self.model_call_details.get("litellm_params")
+        if not isinstance(litellm_params, dict):
+            return
+        metadata = litellm_params.get("metadata")
+        if not isinstance(metadata, dict):
+            return
+        otel_internal = metadata.get("_otel_internal")
+        if not isinstance(otel_internal, dict):
+            return
+        spans_logged = otel_internal.get("spans_logged")
+        if not isinstance(spans_logged, dict):
+            return
+        for scope in [key for key in spans_logged if isinstance(key, tuple) and key[-1:] == ("success",)]:
+            del spans_logged[scope]
+
     def _flush_passthrough_collected_chunks_helper(
         self,
         raw_bytes: list[bytes],
@@ -2282,7 +2375,9 @@ class Logging(LiteLLMLoggingBaseClass):
         is_sync_request: Final = self._is_sync_litellm_request(litellm_params)
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
-            complete_streaming_response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None = None
+            complete_streaming_response: (
+                ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+            ) = None
             if "complete_streaming_response" in self.model_call_details:
                 return  # break out of this.
             complete_streaming_response = self._get_assembled_streaming_response(
@@ -2768,14 +2863,14 @@ class Logging(LiteLLMLoggingBaseClass):
         ## BUILD COMPLETE STREAMED RESPONSE
         if "async_complete_streaming_response" in self.model_call_details:
             return  # break out of this.
-        complete_streaming_response: Final[ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None] = (
-            self._get_assembled_streaming_response(
-                result=result,
-                start_time=start_time,
-                end_time=end_time,
-                is_async=True,
-                streaming_chunks=self.streaming_chunks,
-            )
+        complete_streaming_response: Final[
+            ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None
+        ] = self._get_assembled_streaming_response(
+            result=result,
+            start_time=start_time,
+            end_time=end_time,
+            is_async=True,
+            streaming_chunks=self.streaming_chunks,
         )
 
         if complete_streaming_response is not None:
@@ -3029,6 +3124,13 @@ class Logging(LiteLLMLoggingBaseClass):
         if not hasattr(self, "model_call_details"):
             self.model_call_details = {}
 
+        if (
+            self.model_call_details.get("log_event_type") == "failed_api_call"
+            and self.model_call_details.get("exception") is exception
+            and self.model_call_details.get("standard_logging_object") is not None
+        ):
+            return start_time, self.model_call_details["end_time"]
+
         self.model_call_details["log_event_type"] = "failed_api_call"
         self.model_call_details["exception"] = exception
         self.model_call_details["traceback_exception"] = (
@@ -3558,7 +3660,7 @@ class Logging(LiteLLMLoggingBaseClass):
         end_time: datetime.datetime,
         is_async: bool,
         streaming_chunks: list[object],
-    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | None:
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse | None:
         if self.stream is not True:
             return None
         if isinstance(result, ModelResponse) or isinstance(result, TextCompletionResponse):
@@ -3583,9 +3685,40 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                 )
             return result.response
+        elif isinstance(result, InteractionsAPIStreamingResponse):
+            return self._assemble_completed_interaction_response(result)
         else:
             return None
 
+    @staticmethod
+    def _assemble_completed_interaction_response(
+        result: InteractionsAPIStreamingResponse,
+    ) -> InteractionsAPIResponse | None:
+        """
+        The Interactions API streaming iterator hands the terminal event to the
+        success handlers: the new schema (Api-Revision: 2026-05-20) emits
+        ``interaction.completed`` carrying the full interaction object, the
+        legacy schema (2026-05-07) emits a chunk with ``status="completed"``
+        and usage on the chunk itself. Build the equivalent non-streaming
+        response so cost calculation and spend tracking see one shape.
+        """
+        if result.event_type == "interaction.completed" and result.interaction is not None:
+            return InteractionsAPIResponse(**result.interaction)
+        if result.status == "completed":
+            return InteractionsAPIResponse(
+                **result.model_dump(
+                    exclude={  # mutable-ok: pydantic types exclude as set[str], which a frozenset does not satisfy
+                        "event_type",
+                        "delta",
+                        "index",
+                        "step",
+                        "interaction_id",
+                        "interaction",
+                    }
+                )
+            )
+        return None
+
     def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse:
         """
         Handles logging for Anthropic messages responses.
@@ -4503,6 +4636,9 @@ def _init_custom_logger_compatible_class(
             _in_memory_loggers.append(gitlab_logger)
             return gitlab_logger
         elif logging_integration == "newrelic":
+            _v2 = _maybe_construct_otel_v2("newrelic", _in_memory_loggers)
+            if _v2 is not None:
+                return _v2
             for callback in _in_memory_loggers:
                 if isinstance(callback, NewRelicLogger):
                     return callback
@@ -4789,7 +4925,11 @@ def get_custom_logger_compatible_class(
                 if isinstance(callback, SMTPEmailLogger):
                     return callback
         elif logging_integration == "newrelic":
+            from litellm.integrations.otel.logger import OpenTelemetryV2
+
             for callback in _in_memory_loggers:
+                if isinstance(callback, OpenTelemetryV2) and callback.callback_name == "newrelic":
+                    return callback
                 if isinstance(callback, NewRelicLogger):
                     return callback
         return None
@@ -5085,6 +5225,8 @@ class StandardLoggingPayloadSetup:
         elif isinstance(usage, dict):
             if ResponseAPILoggingUtils._is_response_api_usage(usage):
                 return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
+            if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
+                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
             return Usage(**usage)
 
         raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
@@ -5111,6 +5253,8 @@ class StandardLoggingPayloadSetup:
         if isinstance(_raw, dict):
             if ResponseAPILoggingUtils._is_response_api_usage(_raw):
                 return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_raw).model_dump()
+            if InteractionsUsageObjectTransformation.is_interactions_usage_object(_raw):
+                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(_raw).model_dump()
             return _raw
         if isinstance(_raw, Usage):
             return _raw.model_dump()
@@ -5318,9 +5462,10 @@ class StandardLoggingPayloadSetup:
         error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
         _llm_provider_in_exception: Final = getattr(original_exception, "llm_provider", "")
 
-        # Get traceback information (first 100 lines)
         traceback_info = traceback_str or ""
-        if original_exception:
+        if original_exception and (
+            litellm.log_client_error_tracebacks or not is_expected_client_error(original_exception)
+        ):
             tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
             if tb:
                 tb_lines: Final = traceback.format_tb(tb)
@@ -5793,11 +5938,15 @@ def get_standard_logging_object_payload(
             response_model_name = final_response_obj.get("model")
 
         # For Azure Model Router, preserve the actual model in the top-level standard
-        # logging payload only when the user has opted in.
+        # logging payload.
+        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
         requested_model: Final = kwargs.get("model")
-        if (
-            isinstance(requested_model, str)
-            and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower())
+        stamped_selected_model: Final = AzureFoundryModelInfo.get_model_router_selected_model(hidden_params)
+        if stamped_selected_model is not None:
+            model_name = stamped_selected_model
+        elif (
+            AzureFoundryModelInfo.is_model_router_call(model=requested_model, hidden_params=hidden_params)
             and isinstance(response_model_name, str)
             and response_model_name
         ):

litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py

@@ -1,6 +1,9 @@
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import Any
 
 from litellm.types.utils import (
+    CompletionTokensDetailsWrapper,
     PromptTokensDetailsWrapper,
     TranscriptionUsageDurationObject,
     TranscriptionUsageTokensObject,
@@ -34,3 +37,127 @@ class TranscriptionUsageObjectTransformation:
                 ),
             )
         return None
+
+
+_INTERACTIONS_MODALITY_FIELDS: Mapping[str, str] = MappingProxyType(
+    {
+        "text": "text_tokens",
+        "audio": "audio_tokens",
+        "image": "image_tokens",
+        "video": "video_tokens",
+        "document": "text_tokens",
+    }
+)
+
+
+def _modality_field(entry: Mapping[str, Any]) -> str | None:
+    return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
+
+
+def _token_count(value: object) -> int:
+    return value if isinstance(value, int) else 0
+
+
+def _modality_token_sums(entries: Sequence[Mapping[str, Any]]) -> Mapping[str, int]:
+    fields = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
+    return MappingProxyType(
+        {
+            field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
+            for field in fields
+        }
+    )
+
+
+def _google_search_query_count(usage_object: Mapping[str, Any]) -> int:
+    return sum(
+        _token_count(entry.get("count"))
+        for entry in tuple(usage_object.get("grounding_tool_count") or ())
+        if isinstance(entry, Mapping) and entry.get("type") == "google_search"  # pyright: ignore[reportUnnecessaryIsInstance]  # provider JSON, not the empty tuple inferred from `or ()`
+    )
+
+
+def _subtract_cached_from_input(
+    input_sums: Mapping[str, int],
+    cached_sums: Mapping[str, int],
+    total_cached_tokens: int,
+) -> Mapping[str, int]:
+    if cached_sums:
+        return MappingProxyType(
+            {field: max(0, tokens - cached_sums.get(field, 0)) for field, tokens in input_sums.items()}
+        )
+    if total_cached_tokens and "text_tokens" in input_sums:
+        return MappingProxyType(
+            {
+                **input_sums,
+                "text_tokens": max(0, input_sums["text_tokens"] - total_cached_tokens),
+            }
+        )
+    return input_sums
+
+
+class InteractionsUsageObjectTransformation:
+    """
+    Maps the Google Interactions API usage block (total_input_tokens,
+    output_tokens_by_modality, ...) into LiteLLM's chat-format ``Usage`` so the
+    generic cost calculator and spend tracking can bill it.
+    """
+
+    @staticmethod
+    def is_interactions_usage_object(usage_object: object) -> bool:
+        if not isinstance(usage_object, dict):
+            return False
+        if "prompt_tokens" in usage_object or "input_tokens" in usage_object:
+            return False
+        return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
+
+    @staticmethod
+    def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
+        input_entries = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
+            usage_object.get("tool_use_tokens_by_modality") or ()
+        )
+        cached_sums = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
+        output_sums = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
+
+        total_cached_tokens = _token_count(usage_object.get("total_cached_tokens"))
+        input_sums = _subtract_cached_from_input(
+            input_sums=_modality_token_sums(input_entries),
+            cached_sums=cached_sums,
+            total_cached_tokens=total_cached_tokens,
+        )
+
+        reasoning_tokens = _token_count(usage_object.get("total_reasoning_tokens")) or _token_count(
+            usage_object.get("total_thought_tokens")
+        )
+        prompt_tokens = _token_count(usage_object.get("total_input_tokens")) + _token_count(
+            usage_object.get("total_tool_use_tokens")
+        )
+        completion_tokens = _token_count(usage_object.get("total_output_tokens")) + reasoning_tokens
+        total_tokens = _token_count(usage_object.get("total_tokens")) or (prompt_tokens + completion_tokens)
+
+        web_search_requests = _google_search_query_count(usage_object)
+        prompt_tokens_details = (
+            PromptTokensDetailsWrapper(
+                cached_tokens=total_cached_tokens or None,
+                web_search_requests=web_search_requests or None,
+                **input_sums,
+            )
+            if input_sums or total_cached_tokens or web_search_requests
+            else None
+        )
+        completion_tokens_details = (
+            CompletionTokensDetailsWrapper(
+                reasoning_tokens=reasoning_tokens or None,
+                **output_sums,
+            )
+            if output_sums or reasoning_tokens
+            else None
+        )
+
+        return Usage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            prompt_tokens_details=prompt_tokens_details,
+            completion_tokens_details=completion_tokens_details,
+            cache_read_input_tokens=total_cached_tokens or None,
+        )

litellm/litellm_core_utils/llm_request_utils.py

@@ -1,8 +1,88 @@
+from collections.abc import Mapping
 from typing import Final
 
 import litellm
 
 
+def _form_field_value(value: object) -> str:
+    if value is True:
+        return "true"
+    if value is False:
+        return "false"
+    return str(value)
+
+
+def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
+    if isinstance(value, Mapping):
+        return tuple(
+            item for subkey, subvalue in value.items() for item in _flatten_form_field(f"{key}[{subkey}]", subvalue)
+        )
+    if isinstance(value, (list, tuple)):
+        return tuple(item for entry in value for item in _flatten_form_field(f"{key}[]", entry))
+    if value is None:
+        return ()
+    serialized: Final = _form_field_value(value)
+    if not serialized:
+        return ()
+    return ((key, serialized),)
+
+
+def _is_form_scalar(value: object) -> bool:
+    return value is not None and not isinstance(value, (Mapping, list, tuple))
+
+
+def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
+    if isinstance(value, Mapping):
+        return tuple(
+            item
+            for subkey, subvalue in value.items()
+            for item in _flatten_form_data_field(f"{key}[{subkey}]", subvalue)
+        )
+    if isinstance(value, (list, tuple)):
+        if all(_is_form_scalar(entry) for entry in value):
+            serialized_fields: Final = tuple(field for entry in value if (field := _form_field_value(entry)))
+            return ((key, serialized_fields),) if serialized_fields else ()
+        return tuple(item for entry in value for item in _flatten_form_data_field(f"{key}[]", entry))
+    if value is None:
+        return ()
+    serialized: Final = _form_field_value(value)
+    if not serialized:
+        return ()
+    return ((key, serialized),)
+
+
+def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
+    """
+    Flatten JSON-shaped bodies into ``(name, value)`` form fields for a ``dict``-backed
+    multipart body, applying ``sources`` in order so a later source wins on a key collision
+    under ``dict.update``. Nested objects become ``key[subkey]`` fields the way the OpenAI SDK
+    serializes them, so provider params reach a multipart request without handing the httpx
+    encoder a nested value it rejects with ``Invalid type for value``. A scalar list becomes a
+    single field carrying a tuple value, which httpx emits as one repeated part per element, so
+    every element survives instead of collapsing to the last under ``dict.update``.
+    """
+    return tuple(
+        pair
+        for source in sources
+        if source is not None
+        for top_key, top_value in source.items()
+        for pair in _flatten_form_data_field(top_key, top_value)
+    )
+
+
+def serialize_multipart_form_fields(data: Mapping[str, object]) -> tuple[tuple[str, tuple[None, str]], ...]:
+    """
+    Encode a JSON-shaped body as OpenAI-SDK-style multipart file-tuples so a file-less
+    request is still sent as multipart/form-data, working around httpx downgrading a
+    file-less ``data=`` payload to application/x-www-form-urlencoded.
+    """
+    return tuple(
+        (key, (None, serialized))
+        for top_key, top_value in data.items()
+        for key, serialized in _flatten_form_field(top_key, top_value)
+    )
+
+
 def _ensure_extra_body_is_safe(extra_body: dict | None) -> dict | None:
     """
     Ensure that the extra_body sent in the request is safe,  otherwise users will see this error

litellm/litellm_core_utils/logging_worker.py

@@ -5,7 +5,7 @@ import asyncio
 import atexit
 import contextvars
 import logging
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Iterator
 from typing import Final
 
 from typing_extensions import TypedDict
@@ -61,6 +61,19 @@ class LoggingWorker:
         # Register cleanup handler to flush remaining events on exit
         atexit.register(self._flush_on_exit)
 
+    @staticmethod
+    def _drain_pending(queue: "asyncio.Queue[LoggingTask]") -> tuple[LoggingTask, ...]:
+        """Pop every task still queued, without awaiting them, so they can be moved to another queue."""
+
+        def _pop_until_empty() -> Iterator[LoggingTask]:
+            while True:
+                try:
+                    yield queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    return
+
+        return tuple(_pop_until_empty())
+
     def _ensure_queue(self) -> None:
         """Initialize the queue if it doesn't exist or if event loop has changed."""
         try:
@@ -69,14 +82,27 @@ class LoggingWorker:
             # No running loop, can't initialize
             return
 
-        # Check if we need to reinitialize due to event loop change
+        # The queue, semaphore and worker task are all bound to the loop that created them. On a
+        # loop change we hand the still-pending tasks to a fresh queue instead of dropping them,
+        # so queued spend-logging coroutines are not silently discarded (and never left un-awaited).
         if self._queue is not None and self._bound_loop is not current_loop:
-            verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
-            # Clear old state - these are bound to the old loop
-            self._queue = None
+            carried_over: Final = self._drain_pending(self._queue)
+            new_queue: Final[asyncio.Queue[LoggingTask]] = asyncio.Queue(maxsize=self.max_queue_size)
+            for carried_task in carried_over:
+                new_queue.put_nowait(carried_task)
+            if carried_over:
+                verbose_logger.warning(
+                    "LoggingWorker: event loop changed; carried %d pending logging task(s) onto the new loop",
+                    len(carried_over),
+                )
+            else:
+                verbose_logger.debug("LoggingWorker: Event loop changed, reinitializing queue and worker")
             self._sem = None
             self._worker_task = None
             self._running_tasks.clear()
+            self._queue = new_queue
+            self._bound_loop = current_loop
+            return
 
         if self._queue is None:
             self._queue = asyncio.Queue(maxsize=self.max_queue_size)

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -28,8 +28,12 @@ from litellm.types.llms.openai import (
     ChatCompletionAssistantMessage,
     ChatCompletionFileObject,
     ChatCompletionImageObject,
+    ChatCompletionReasoningItem,
+    ChatCompletionReasoningSummaryTextBlock,
+    ChatCompletionRedactedThinkingBlock,
     ChatCompletionResponseMessage,
     ChatCompletionTextObject,
+    ChatCompletionThinkingBlock,
     ChatCompletionToolParam,
     ChatCompletionUserMessage,
 )
@@ -466,6 +470,8 @@ def update_messages_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     for message in messages:
@@ -504,6 +510,11 @@ def update_messages_with_model_file_ids(
                                 unified_file_id = convert_b64_uid_to_unified_uid(file_id)
                                 if "llm_output_file_id," in unified_file_id:
                                     provider_file_id = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
+                            if not provider_file_id and is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                provider_file_id = get_original_file_id(file_id)
                             file_object_file_field["file_id"] = provider_file_id or file_id
                         if format:
                             file_object_file_field["format"] = format
@@ -531,6 +542,8 @@ def update_responses_input_with_model_file_ids(
     from litellm.proxy.openai_files_endpoints.common_utils import (
         _is_base64_encoded_unified_file_id,
         convert_b64_uid_to_unified_uid,
+        get_original_file_id,
+        is_model_embedded_id,
     )
 
     if isinstance(input, str):
@@ -574,6 +587,13 @@ def update_responses_input_with_model_file_ids(
                                 updated_content_item = content_item.copy()
                                 updated_content_item["file_id"] = provider_file_id
                                 updated_content.append(updated_content_item)
+                            elif is_model_embedded_id(file_id):
+                                # `litellm:<raw_id>;model,<m>` encoding from the
+                                # x-litellm-model upload path. Strip the wrapper
+                                # so the provider sees its own ID.
+                                updated_content_item = content_item.copy()
+                                updated_content_item["file_id"] = get_original_file_id(file_id)
+                                updated_content.append(updated_content_item)
                             else:
                                 # Not a managed file, keep as-is
                                 updated_content.append(content_item)
@@ -1549,6 +1569,44 @@ def _extract_reasoning_content(message: dict) -> tuple[str | None, str | None]:
     return None, message_content
 
 
+def _readable_thinking_text(
+    block: ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock,
+) -> str:
+    """The text a chat model can read back, empty for redacted blocks and malformed ones."""
+    if block.get("type") != "thinking":
+        return ""
+    thinking: Final = cast(ChatCompletionThinkingBlock, block).get("thinking")  # cast-ok: narrowed by the type tag
+    return str(thinking or "")
+
+
+def reasoning_content_from_thinking_blocks(
+    thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
+) -> str:
+    """Flatten Anthropic thinking blocks into the `reasoning_content` string chat models expect.
+
+    Redacted blocks carry no readable text, so they contribute nothing.
+    """
+    return "\n".join(text for block in thinking_blocks if (text := _readable_thinking_text(block)))
+
+
+def responses_reasoning_item_from_thinking_blocks(
+    thinking_blocks: Iterable[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
+) -> ChatCompletionReasoningItem | None:
+    """Build a Responses API `reasoning` input item from Anthropic thinking blocks.
+
+    The item carries no `id`: the Responses API rejects an empty one and 404s on any id it
+    did not mint itself, while an item without an id is always accepted.
+    """
+    summary: Final[list[ChatCompletionReasoningSummaryTextBlock]] = [  # mutable-ok: API message payload
+        ChatCompletionReasoningSummaryTextBlock(type="summary_text", text=text)
+        for block in thinking_blocks
+        if (text := _readable_thinking_text(block))
+    ]
+    if not summary:
+        return None
+    return ChatCompletionReasoningItem(type="reasoning", summary=summary)
+
+
 def _parse_content_for_reasoning(
     message_text: str | None,
 ) -> tuple[str | None, str | None]:

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -16,6 +16,7 @@ import litellm.types
 import litellm.types.llms
 from litellm import verbose_logger
 from litellm._uuid import uuid
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
 from litellm.types.files import get_file_extension_from_mime_type
@@ -642,49 +643,6 @@ def claude_2_1_pt(
     return prompt
 
 
-### TOGETHER AI
-
-
-def get_model_info(token, model):
-    try:
-        headers: Final = {"Authorization": f"Bearer {token}"}
-        client: Final = HTTPHandler(concurrent_limit=1)
-        response: Final = client.get("https://api.together.xyz/models/info", headers=headers)
-        if response.status_code == 200:
-            model_info: Final = response.json()
-            for m in model_info:
-                if m["name"].lower().strip() == model.strip():
-                    return m["config"].get("prompt_format", None), m["config"].get("chat_template", None)
-            return None, None
-        else:
-            return None, None
-    except Exception:  # safely fail a prompt template request
-        return None, None
-
-
-## OLD TOGETHER AI FLOW
-# def format_prompt_togetherai(messages, prompt_format, chat_template):
-#     if prompt_format is None:
-#         return default_pt(messages)
-
-#     human_prompt, assistant_prompt = prompt_format.split("{prompt}")
-
-#     if chat_template is not None:
-#         prompt = hf_chat_template(
-#             model=None, messages=messages, chat_template=chat_template
-#         )
-#     elif prompt_format is not None:
-#         prompt = custom_prompt(
-#             role_dict={},
-#             messages=messages,
-#             initial_prompt_value=human_prompt,
-#             final_prompt_value=assistant_prompt,
-#         )
-#     else:
-#         prompt = default_pt(messages)
-#     return prompt
-
-
 ### IBM Granite
 
 
@@ -5383,12 +5341,13 @@ def _parse_tool_call_arguments(raw: Any, tool_name: str | None, context: str) ->
         return raw
     if not isinstance(raw, str):
         return {}
+    normalized_raw: Final = "{}" if raw == REDACTED_BY_LITELLM else raw
     from litellm.litellm_core_utils.prompt_templates.common_utils import (
         parse_tool_call_arguments,
     )
 
     try:
-        parsed: Final = parse_tool_call_arguments(raw, tool_name=tool_name, context=context)
+        parsed: Final = parse_tool_call_arguments(normalized_raw, tool_name=tool_name, context=context)
     except ValueError as e:
         verbose_logger.warning("Failed to parse tool call arguments: %s", e)
         return {}

litellm/litellm_core_utils/redact_messages.py

@@ -13,6 +13,7 @@ import inspect
 from typing import TYPE_CHECKING, Any, Final
 
 import litellm
+from litellm.constants import REDACTED_BY_LITELLM
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
@@ -84,29 +85,31 @@ def _redact_tool_calls(tool_calls) -> None:
     for tool_call in tool_calls:
         function = getattr(tool_call, "function", None)
         if function is not None and hasattr(function, "arguments"):
-            function.arguments = "redacted-by-litellm"
+            function.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_function_call(function_call) -> None:
     """Redact legacy assistant function_call arguments."""
     if function_call is not None and hasattr(function_call, "arguments"):
-        function_call.arguments = "redacted-by-litellm"
+        function_call.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
-        choice.message.content = "redacted-by-litellm"
-        if hasattr(choice.message, "reasoning_content"):
-            choice.message.reasoning_content = "redacted-by-litellm"
+        if choice.message.content is not None:
+            choice.message.content = REDACTED_BY_LITELLM
+        if getattr(choice.message, "reasoning_content", None) is not None:
+            choice.message.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
         _redact_tool_calls(getattr(choice.message, "tool_calls", None))
         _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
-        choice.delta.content = "redacted-by-litellm"
-        if hasattr(choice.delta, "reasoning_content"):
-            choice.delta.reasoning_content = "redacted-by-litellm"
+        if choice.delta.content is not None:
+            choice.delta.content = REDACTED_BY_LITELLM
+        if getattr(choice.delta, "reasoning_content", None) is not None:
+            choice.delta.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
         _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
@@ -116,23 +119,23 @@ def _redact_choice_content(choice):
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
-        if hasattr(output_item, "text"):
-            output_item.text = "redacted-by-litellm"
+        if getattr(output_item, "text", None) is not None:
+            output_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
-                if hasattr(content_part, "text"):
-                    content_part.text = "redacted-by-litellm"
+                if getattr(content_part, "text", None) is not None:
+                    content_part.text = REDACTED_BY_LITELLM
 
         # Redact reasoning items in output array
         if hasattr(output_item, "type") and output_item.type == "reasoning":
             if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
                 for summary_item in output_item.summary:
-                    if hasattr(summary_item, "text"):
-                        summary_item.text = "redacted-by-litellm"
+                    if getattr(summary_item, "text", None) is not None:
+                        summary_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
-            output_item.arguments = "redacted-by-litellm"
+            output_item.arguments = REDACTED_BY_LITELLM
 
 
 def _redact_responses_api_output_dict(output_items, redacted_str: str):
@@ -141,17 +144,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
         if not isinstance(output_item, dict):
             continue
 
-        if "text" in output_item:
+        if output_item.get("text") is not None:
             output_item["text"] = redacted_str
 
         if isinstance(output_item.get("content"), list):
             for content_item in output_item["content"]:
-                if isinstance(content_item, dict) and "text" in content_item:
+                if isinstance(content_item, dict) and content_item.get("text") is not None:
                     content_item["text"] = redacted_str
 
         if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
             for summary_item in output_item["summary"]:
-                if isinstance(summary_item, dict) and "text" in summary_item:
+                if isinstance(summary_item, dict) and summary_item.get("text") is not None:
                     summary_item["text"] = redacted_str
 
         if output_item.get("type") == "function_call" and "arguments" in output_item:
@@ -164,7 +167,7 @@ def _redact_standard_logging_object(model_call_details: dict):
     if standard_logging_object is None:
         return
 
-    redacted_str: Final = "redacted-by-litellm"
+    redacted_str: Final = REDACTED_BY_LITELLM
 
     if standard_logging_object.get("messages") is not None:
         standard_logging_object["messages"] = [{"role": "user", "content": redacted_str}]
@@ -188,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
-def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
+def _redact_tool_calls_dict(message: dict) -> None:
     """Redact tool call / function_call arguments in a dict-form message or delta."""
     tool_calls: Final = message.get("tool_calls")
     if isinstance(tool_calls, list):
         for tool_call in tool_calls:
             if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
-                tool_call["function"]["arguments"] = redacted_str
+                tool_call["function"]["arguments"] = REDACTED_BY_LITELLM
 
     function_call: Final = message.get("function_call")
     if isinstance(function_call, dict) and "arguments" in function_call:
-        function_call["arguments"] = redacted_str
+        function_call["arguments"] = REDACTED_BY_LITELLM
 
 
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
             if "message" in choice and isinstance(choice["message"], dict):
-                choice["message"]["content"] = redacted_str
-                if "reasoning_content" in choice["message"]:
+                if choice["message"].get("content") is not None:
+                    choice["message"]["content"] = redacted_str
+                if choice["message"].get("reasoning_content") is not None:
                     choice["message"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["message"]:
                     choice["message"]["thinking_blocks"] = None
                 if "audio" in choice["message"]:
                     choice["message"]["audio"] = None
-                _redact_tool_calls_dict(choice["message"], redacted_str)
+                _redact_tool_calls_dict(choice["message"])
             elif "delta" in choice and isinstance(choice["delta"], dict):
-                choice["delta"]["content"] = redacted_str
-                if "reasoning_content" in choice["delta"]:
+                if choice["delta"].get("content") is not None:
+                    choice["delta"]["content"] = redacted_str
+                if choice["delta"].get("reasoning_content") is not None:
                     choice["delta"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["delta"]:
                     choice["delta"]["thinking_blocks"] = None
                 if "audio" in choice["delta"]:
                     choice["delta"]["audio"] = None
-                _redact_tool_calls_dict(choice["delta"], redacted_str)
+                _redact_tool_calls_dict(choice["delta"])
         else:
             _redact_choice_content(choice)
 
@@ -235,7 +240,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
     copy via redact_streaming_responses_for_custom_logger instead.
     """
     # Redact model_call_details
-    model_call_details["messages"] = [{"role": "user", "content": "redacted-by-litellm"}]
+    model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""
     _redact_standard_logging_object(model_call_details)
@@ -256,13 +261,13 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             or hasattr(result, "__anext__")  # async generator
         ):  # async iterator
             # For async objects, return a simple redacted response without deepcopy
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         if not (
             isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
             or (isinstance(result, dict) and ("choices" in result or "output" in result))
         ):
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         _result: Final = copy.deepcopy(result)
         if isinstance(_result, litellm.ModelResponse):
@@ -273,11 +278,11 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
         elif isinstance(_result, dict) and "choices" in _result:
             # Handle dict representation of ModelResponse (e.g., from model_dump())
             if _result.get("choices") is not None:
-                _redact_model_response_dict_choices(_result["choices"], "redacted-by-litellm")
+                _redact_model_response_dict_choices(_result["choices"], REDACTED_BY_LITELLM)
             redact_vertex_ai_metadata_from_logged_object(_result)
         elif isinstance(_result, dict) and "output" in _result:
             if isinstance(_result.get("output"), list):
-                _redact_responses_api_output_dict(_result["output"], "redacted-by-litellm")
+                _redact_responses_api_output_dict(_result["output"], REDACTED_BY_LITELLM)
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)
@@ -288,7 +293,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             if hasattr(_result, "data") and _result.data is not None:
                 _result.data = []
         else:
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
         return _result
 
 

litellm/llms/anthropic/chat/handler.py

@@ -18,6 +18,7 @@ from litellm.anthropic_beta_headers_manager import (
 )
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -654,7 +655,7 @@ class ModelResponseIterator:
 
         # For handling partial JSON chunks from fragmentation
         # See: https://github.com/BerriAI/litellm/issues/17473
-        self.accumulated_json: str = ""
+        self._json_buffer = JSONFragmentAccumulator()
         self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
 
         # Track current content block type to avoid emitting tool calls for non-tool blocks
@@ -678,6 +679,14 @@ class ModelResponseIterator:
         self._current_server_tool_id: str | None = None
         self._container_id: str | None = None
 
+    @property
+    def accumulated_json(self) -> str:
+        return self._json_buffer.snapshot()
+
+    @accumulated_json.setter
+    def accumulated_json(self, value: str) -> None:
+        self._json_buffer.set(value)
+
     def check_empty_tool_call_args(self) -> bool:
         """
         Check if the tool call block so far has been an empty string
@@ -1149,31 +1158,39 @@ class ModelResponseIterator:
         container: Final = message_delta["delta"].get("container")
         return finish_reason, usage, container
 
-    def _handle_accumulated_json_chunk(self, data_str: str) -> ModelResponseStream | None:
+    def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
         """
         Handle partial JSON chunks by accumulating them until valid JSON is received.
 
         This fixes network fragmentation issues where SSE data chunks may be split
         across TCP packets. See: https://github.com/BerriAI/litellm/issues/17473
 
+        Mid-stream, defer parsing until the buffer's last byte can close a value:
+        attempting a parse after every fragment of one large object is O(n^2) and
+        holds the GIL, freezing the event loop. At end of stream (is_final) no more
+        data is coming, so drain whatever complete values remain regardless of the
+        trailing byte.
+
         Args:
             data_str: The JSON string to parse (without "data:" prefix)
+            is_final: True when called from the end-of-stream drain, where the
+                trailing-byte heuristic no longer applies
 
         Returns:
             ModelResponseStream if JSON is complete, None if still accumulating
         """
-        # Accumulate JSON data
-        self.accumulated_json += data_str
+        self._json_buffer.append(data_str)
 
-        # Try to parse the accumulated JSON
-        try:
-            data_json: Final = json.loads(self.accumulated_json)
-            self.accumulated_json = ""  # Reset after successful parsing
-            return self.chunk_parser(chunk=data_json)
-        except json.JSONDecodeError:
-            # If it's not valid JSON yet, continue to the next chunk
+        if not is_final and not self._json_buffer.could_close_json():
             return None
 
+        while True:
+            found, decoded = self._json_buffer.pop_next_value()
+            if not found:
+                return None
+            if isinstance(decoded, dict):
+                return self.chunk_parser(chunk=decoded)
+
     def _parse_sse_data(self, str_line: str) -> ModelResponseStream | None:
         """
         Parse SSE data line, handling both complete and partial JSON chunks.
@@ -1209,13 +1226,10 @@ class ModelResponseIterator:
                 chunk = self.response_iterator.__next__()
             except StopIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")
@@ -1258,13 +1272,10 @@ class ModelResponseIterator:
                 chunk = await self.async_response_iterator.__anext__()
             except StopAsyncIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopAsyncIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")

litellm/llms/anthropic/chat/transformation.py

@@ -1,7 +1,8 @@
 import json
 import re
 import time
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, NoReturn, cast
 
 import httpx
@@ -121,6 +122,32 @@ else:
 # response side.
 _ANTHROPIC_TOOL_NAME_INVALID_CHARS: Final = re.compile(r"[^a-zA-Z0-9_-]")
 _ANTHROPIC_TOOL_NAME_MAX_LEN: Final = 128
+
+_ENUM_TYPE_CHECKS: Final[Mapping[str, Callable[[Any], bool]]] = MappingProxyType(
+    {
+        "null": lambda v: v is None,
+        "boolean": lambda v: isinstance(v, bool),
+        "integer": lambda v: isinstance(v, int) and not isinstance(v, bool),
+        "number": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
+        "string": lambda v: isinstance(v, str),
+        "array": lambda v: isinstance(v, list),
+        "object": lambda v: isinstance(v, dict),
+    }
+)
+
+
+def _enum_conflicts_with_declared_type(schema: Mapping[str, Any]) -> bool:
+    """Whether ``schema``'s ``enum`` cannot match its declared ``type``."""
+    enum_values: Final = schema.get("enum")
+    declared_type: Final = schema.get("type")
+    if not isinstance(enum_values, list) or declared_type is None:
+        return False
+    if isinstance(declared_type, list):
+        return True
+    check: Final = _ENUM_TYPE_CHECKS.get(declared_type)
+    return check is not None and not all(check(value) for value in enum_values)
+
+
 # Single, internal-only key on ``litellm_params`` used to thread the per-
 # request reverse map (sanitized -> original) from request build to response
 # parsing. ``litellm_params`` is never serialized to a provider; ``optional_
@@ -565,9 +592,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             else:
                 result["description"] = constraint_note
 
+        drops_conflicting_type: Final = _enum_conflicts_with_declared_type(schema)
+
         for key, value in schema.items():
             if key in unsupported_fields:
                 continue
+            if key == "type" and drops_conflicting_type:
+                continue
             if key == "description" and "description" in result:
                 # Already handled above
                 continue
@@ -1184,8 +1215,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if reasoning_effort is None or reasoning_effort == "none":
             return None
         if AnthropicConfig._is_adaptive_thinking_model(model, custom_llm_provider):
+            # without display, Anthropic defaults adaptive thinking to
+            # display="omitted" and returns a blank thinking block
             return AnthropicThinkingParam(
                 type="adaptive",
+                display="summarized",
             )
         elif reasoning_effort == "low":
             return AnthropicThinkingParam(
@@ -2113,7 +2147,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         )
 
     @staticmethod
-    def _thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
+    def thinking_tokens_from_usage(usage_object: Mapping[str, object]) -> int | None:
         details: Final = usage_object.get("output_tokens_details")
         if not isinstance(details, Mapping):
             return None
@@ -2145,7 +2179,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         reported_thinking_tokens: Final = (
             iteration_thinking_tokens
             if iteration_thinking_tokens is not None
-            else self._thinking_tokens_from_usage(usage_object)
+            else self.thinking_tokens_from_usage(usage_object)
         )
         if reported_thinking_tokens is not None:
             capped_reported: Final = min(max(0, reported_thinking_tokens), completion_tokens)
@@ -2168,7 +2202,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def _sum_iteration_thinking_tokens(self, iterations: Sequence[object]) -> int | None:
         per_iteration: Final = tuple(
-            self._thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
+            self.thinking_tokens_from_usage(iteration) if isinstance(iteration, Mapping) else None
             for iteration in iterations
         )
         reported: Final = tuple(tokens for tokens in per_iteration if tokens is not None)

litellm/llms/anthropic/common_utils.py

@@ -38,6 +38,21 @@ DROP_DISABLED_THINKING_WARNING: Final = (
     "thinking blocks, and those thinking tokens are billed as output tokens."
 )
 
+# Anthropic error `type` (both the JSON error body and SSE `event: error`
+# payloads use this field) mapped to the HTTP status code it corresponds to.
+ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = MappingProxyType(
+    {
+        "invalid_request_error": 400,
+        "authentication_error": 401,
+        "permission_error": 403,
+        "not_found_error": 404,
+        "rate_limit_error": 429,
+        "api_error": 500,
+        "overloaded_error": 503,
+        "timeout_error": 504,
+    }
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
@@ -440,6 +455,16 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         """
         return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
 
+    @staticmethod
+    def _supports_legacy_thinking(model: str, custom_llm_provider: str) -> bool:
+        """Whether ``model`` is an adaptive-thinking model that still accepts legacy
+        ``thinking.type=enabled`` with ``budget_tokens`` (the Claude 4.6 family).
+        The model cost map is authoritative: an explicit ``supports_legacy_thinking``
+        entry resolved under ``custom_llm_provider``, or a ``fallback_generalizations``
+        rule for unmapped 4.6 ids. Absent flag means the model rejects the legacy shape.
+        """
+        return AnthropicModelInfo._supports_model_capability(model, "supports_legacy_thinking", custom_llm_provider)
+
     @staticmethod
     def maybe_drop_disabled_thinking(
         model: str,

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -64,6 +64,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     parse_tool_call_arguments,
+    reasoning_content_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -433,7 +434,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                 content_items = list(content.get("content", []))
 
                                 # Single-item text keeps the backward-compatible string format; a single
-                                # image becomes a structured image_url part
+                                # image or document becomes a structured image_url part
                                 if len(content_items) == 1:
                                     c = content_items[0]
                                     if isinstance(c, str):
@@ -453,7 +454,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                             )
                                             self._add_cache_control_if_applicable(content, tool_result, model)
                                             tool_message_list.append(tool_result)
-                                        elif c.get("type") == "image":
+                                        elif c.get("type") in ("image", "document"):
                                             image_part = self._tool_result_image_part(c.get("source"))
                                             tool_result = ChatCompletionToolMessage(
                                                 role="tool",
@@ -481,7 +482,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                                         text=c.get("text", ""),
                                                     )
                                                 )
-                                            elif c.get("type") == "image":
+                                            elif c.get("type") in ("image", "document"):
                                                 image_part = self._tool_result_image_part(c.get("source"))
                                                 if image_part:
                                                     combined_content_parts.append(image_part)
@@ -592,6 +593,9 @@ class LiteLLMAnthropicMessagesAdapter:
                     assistant_message["tool_calls"] = tool_calls
                 if len(thinking_blocks) > 0:
                     assistant_message["thinking_blocks"] = thinking_blocks
+                reasoning_content = reasoning_content_from_thinking_blocks(thinking_blocks)
+                if reasoning_content:
+                    assistant_message["reasoning_content"] = reasoning_content
                 new_messages.append(assistant_message)
 
         return new_messages

litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py

@@ -6,13 +6,41 @@ yields every chunk to the caller (preserving real streaming), collects
 all bytes, and on stream exhaustion rebuilds the full Anthropic response
 to run through agentic completion hooks. If an agentic hook fires, the
 follow-up response is chained as Phase 2 of the same iterator.
+
+In hold-back mode (``hold_back=True``) chunks are buffered instead of yielded
+live, keepalive pings run whenever no other byte is ready, and then either the
+follow-up replaces the message or the buffer replays, except that a tool_use for
+a server-fulfilled tool fails the turn rather than reaching a client that cannot
+execute it.
 """
 
+import asyncio
+import contextlib
 import json
 from collections.abc import AsyncIterator
 from typing import Any, Final, cast
 
 from litellm._logging import verbose_logger
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
+
+HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
+SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
+    b"event: error\n"
+    b'data: {"type": "error", "error": {"type": "api_error", "message": '
+    b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
+)
+
+
+def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
+    return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
+
+async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
+    try:
+        return await iterator.__anext__()
+    except StopAsyncIteration:
+        return None
+
 
 # ---------------------------------------------------------------------------
 # SSE parsing helpers (module-level to keep the class lean)
@@ -156,6 +184,9 @@ class AgenticAnthropicStreamingIterator:
         logging_obj: Any,
         custom_llm_provider: str,
         kwargs: dict,
+        hold_back: bool = False,
+        server_fulfilled_tool_names: frozenset[str] = frozenset(),
+        ping_interval_seconds: float = HOLD_BACK_PING_INTERVAL_SECONDS,
     ):
         self._inner = completion_stream.__aiter__()
         self._http_handler = http_handler
@@ -166,16 +197,32 @@ class AgenticAnthropicStreamingIterator:
         self._logging_obj = logging_obj
         self._custom_llm_provider = custom_llm_provider
         self._kwargs = kwargs
+        self._hold_back = hold_back
+        self._server_fulfilled_tool_names = server_fulfilled_tool_names
+        self._ping_interval_seconds = ping_interval_seconds
 
         self._collected_bytes: list[bytes] = []
         self._stream_exhausted = False
         self._hook_processing_done = False
         self._follow_up_iterator: AsyncIterator | None = None
+        self._drain_task: asyncio.Task | None = None
+        self._hook_task: asyncio.Task | None = None
+        self._follow_up_chunk_task: asyncio.Task | None = None
+        self._replay_index = 0
+        self._error_emitted = False
+
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        """Whether provider output was received but withheld from the client behind keepalive pings."""
+        return self._hold_back and bool(self._collected_bytes)
 
     def __aiter__(self):
         return self
 
     async def __anext__(self) -> bytes:
+        if self._hold_back:
+            return await self._anext_held_back()
+
         # Phase 1: yield from upstream, collect bytes
         if not self._stream_exhausted:
             try:
@@ -194,11 +241,102 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    async def _drain_upstream(self) -> None:
+        try:
+            while True:
+                self._collected_bytes.append(await self._inner.__anext__())
+        except StopAsyncIteration:
+            return
+
+    async def _completed_within_ping_interval(self, task: asyncio.Task) -> bool:
+        try:
+            await asyncio.wait_for(asyncio.shield(task), timeout=self._ping_interval_seconds)
+        except asyncio.TimeoutError:
+            return False
+        return True
+
+    async def _anext_held_back(self) -> bytes:
+        if self._drain_task is None:
+            self._drain_task = asyncio.create_task(self._drain_upstream())
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+
+        if not self._stream_exhausted:
+            if not await self._completed_within_ping_interval(self._drain_task):
+                return STREAM_SSE_KEEPALIVE_PING_BYTES
+            self._stream_exhausted = True
+
+        if self._hook_task is None:
+            self._hook_task = asyncio.create_task(self._process_agentic_hooks())
+        if not await self._completed_within_ping_interval(self._hook_task):
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+
+        if self._follow_up_iterator is not None:
+            return await self._next_follow_up_chunk(self._follow_up_iterator)
+
+        if self._buffer_holds_server_fulfilled_tool_use():
+            if self._error_emitted:
+                raise StopAsyncIteration
+            self._error_emitted = True
+            verbose_logger.error(
+                "AgenticStreamingIterator: hooks did not replace a message containing a server-fulfilled "
+                "tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
+                self._model,
+            )
+            return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
+        if self._replay_index < len(self._collected_bytes):
+            chunk: Final = self._collected_bytes[self._replay_index]
+            self._replay_index += 1
+            return chunk
+
+        raise StopAsyncIteration
+
+    async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
+        if self._follow_up_chunk_task is None:
+            self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
+        if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
+        chunk: Final = self._follow_up_chunk_task.result()
+        self._follow_up_chunk_task = None
+        if chunk is None:
+            raise StopAsyncIteration
+        return chunk
+
+    def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
+        if not self._server_fulfilled_tool_names:
+            return False
+        started_blocks: Final = (
+            data.get("content_block")
+            for event_type, data in _parse_sse_events(b"".join(self._collected_bytes))
+            if event_type == "content_block_start"
+        )
+        return any(
+            isinstance(block, dict)
+            and block.get("type") == "tool_use"
+            and block.get("name") in self._server_fulfilled_tool_names
+            for block in started_blocks
+        )
+
+    @staticmethod
+    async def _settle_task(task: asyncio.Task | None) -> None:
+        if task is None:
+            return
+        if task.done():
+            if not task.cancelled():
+                task.exception()
+            return
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+
     async def aclose(self) -> None:
         from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
             aclose_if_supported,
         )
 
+        await self._settle_task(self._drain_task)
+        await self._settle_task(self._hook_task)
+        await self._settle_task(self._follow_up_chunk_task)
         await aclose_if_supported(self._inner)
         await aclose_if_supported(self._follow_up_iterator)
 
@@ -217,11 +355,6 @@ class AgenticAnthropicStreamingIterator:
                 verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
                 return
 
-            [
-                (f"{b.get('type')}({b.get('name', '')})" if b.get("type") == "tool_use" else b.get("type"))
-                for b in rebuilt.get("content", [])
-            ]
-
             result: Final = await self._http_handler._call_agentic_completion_hooks(
                 response=rebuilt,
                 model=self._model,

litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py

@@ -46,6 +46,10 @@ class AnthropicMessagesStreamCacheWriter:
             stream._hidden_params if isinstance(stream, AnthropicMessagesStreamingResponse) else _EMPTY_MAPPING
         )
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamCacheWriter":
         return self
 

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from datetime import datetime
 from typing import Any, Final, Protocol, runtime_checkable
 
@@ -11,9 +11,11 @@ from typing_extensions import TypedDict
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
@@ -33,26 +35,239 @@ def _is_message_stop_chunk(chunk: object) -> bool:
     return False
 
 
-def _is_provider_error_chunk(chunk: object) -> bool:
+def is_anthropic_ping_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk is a pure ``ping`` keepalive frame. It carries no content
+    and can recur indefinitely on a slow-starting or idle connection, so a
+    mid-stream fallback wrapper drops it outright while still deciding
+    whether to commit to the primary stream, rather than buffering it.
+
+    A physical transport chunk that coalesces a ping with any other SSE
+    event (``message_start``, ``content_block_delta``, ``event: error``, ...)
+    is NOT a pure ping - dropping it whole would discard those events - so
+    only a chunk whose every ``event:`` line is ``event: ping`` qualifies.
+    """
     if isinstance(chunk, dict):
-        return chunk.get("type") == "error"
+        return chunk.get("type") == "ping"
     if isinstance(chunk, (bytes, bytearray)):
-        return any(line == b"event: error" for line in chunk.splitlines())
+        event_lines: Final = tuple(line for line in chunk.splitlines() if line.startswith(b"event:"))
+        return bool(event_lines) and all(line == b"event: ping" for line in event_lines)
     return False
 
 
+def is_anthropic_content_delta_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk carries actual assistant-generated output (a
+    ``content_block_delta`` frame), as opposed to a lifecycle/bookkeeping
+    frame (``message_start``, ``content_block_start``/``stop``,
+    ``message_delta``, ``message_stop``, ``ping``) that carries nothing
+    worth preserving before an invisible mid-stream fallback retry.
+    """
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "content_block_delta"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: content_block_delta" for line in chunk.splitlines())
+    return False
+
+
+def _decoded_sse_data_line(line: bytes) -> object | None:
+    if not line.startswith(b"data:"):
+        return None
+    try:
+        return json.loads(line[len(b"data:") :].strip())
+    except (ValueError, TypeError):
+        return None
+
+
+def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
+    if isinstance(chunk, dict):
+        return chunk if chunk.get("type") == "error" else None
+    if isinstance(chunk, (bytes, bytearray)):
+        decoded_lines: Final = (_decoded_sse_data_line(line) for line in chunk.splitlines())
+        return next(
+            (
+                candidate
+                for candidate in decoded_lines
+                if isinstance(candidate, dict) and candidate.get("type") == "error"
+            ),
+            None,
+        )
+    return None
+
+
+def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
+    """Return the ``error`` object of an Anthropic SSE ``event: error`` chunk, or None."""
+    payload: Final = _anthropic_error_event_payload(chunk)
+    error_body: Final = payload.get("error") if payload is not None else None
+    return error_body if isinstance(error_body, dict) else None
+
+
+def _is_provider_error_chunk(chunk: object) -> bool:
+    return _anthropic_error_body(chunk) is not None
+
+
+def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
+    """
+    Extract ``(error_type, message, http_status_code)`` from an Anthropic SSE
+    ``event: error`` chunk (raw bytes or an already-decoded dict), or None if
+    ``chunk`` is not an error event.
+
+    The status code is looked up via ANTHROPIC_ERROR_STATUS_CODE_MAP,
+    defaulting to 500 for an error ``type`` Anthropic hasn't documented yet.
+    """
+    error_body: Final = _anthropic_error_body(chunk)
+    if error_body is None:
+        return None
+    error_type: Final = error_body.get("type")
+    if not isinstance(error_type, str):
+        return None
+    message: Final = error_body.get("message")
+    return (
+        error_type,
+        message if isinstance(message, str) else error_type,
+        ANTHROPIC_ERROR_STATUS_CODE_MAP.get(error_type, 500),
+    )
+
+
 def _is_terminal_stream_chunk(chunk: object) -> bool:
     return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
 
 
+def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
+    return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
 def _incomplete_stream_error_sse_event() -> bytes:
-    payload: Final = json.dumps(
-        {
-            "type": "error",
-            "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
-        }
+    return _sse_event(  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "error",
+        {"type": "error", "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}},
+    )
+
+
+def _anthropic_content_block_start_and_deltas(
+    block: Mapping[str, object],
+) -> tuple[Mapping[str, object], tuple[Mapping[str, object], ...]]:
+    """
+    ``(content_block_start.content_block, content_block_delta.delta events)``
+    for one Anthropic response content block. A thinking block emits both a
+    thinking_delta and a trailing signature_delta - a real Anthropic stream
+    does the same, and dropping the signature makes any replay of that
+    assistant message (a follow-up turn, a tool-use continuation) fail
+    Anthropic's thinking-signature verification. redacted_thinking has no
+    delta at all - it is sent complete in content_block_start.
+    """
+    match block.get("type"):
+        case "tool_use":
+            return (
+                {  # mutable-ok: one-shot payload
+                    "id": block.get("id"),
+                    "name": block.get("name"),
+                    "input": {},  # mutable-ok: one-shot payload
+                    "type": "tool_use",
+                },
+                (
+                    {  # mutable-ok: one-shot payload
+                        "partial_json": json.dumps(block.get("input") or {}),  # mutable-ok: one-shot payload
+                        "type": "input_json_delta",
+                    },
+                ),
+            )
+        case "thinking":
+            signature: Final = block.get("signature")
+            signature_deltas: Final = (
+                ({"signature": signature, "type": "signature_delta"},)  # mutable-ok: one-shot payload
+                if isinstance(signature, str) and signature
+                else ()
+            )
+            return (
+                {"thinking": "", "signature": "", "type": "thinking"},  # mutable-ok: one-shot payload
+                (
+                    {"thinking": block.get("thinking") or "", "type": "thinking_delta"},  # mutable-ok: one-shot payload
+                    *signature_deltas,
+                ),
+            )
+        case "redacted_thinking":
+            return ({"type": "redacted_thinking", "data": block.get("data")}, ())  # mutable-ok: one-shot JSON payload
+        case _:
+            return (
+                {"type": "text", "text": ""},  # mutable-ok: one-shot JSON payload
+                ({"type": "text_delta", "text": block.get("text") or ""},),  # mutable-ok: one-shot JSON payload
+            )
+
+
+def anthropic_messages_response_as_sse_events(response: AnthropicMessagesResponse) -> tuple[bytes, ...]:
+    """
+    Render a complete (non-streaming) AnthropicMessagesResponse as the SSE
+    event sequence a real streaming request would have produced.
+
+    A mid-stream fallback can resolve to a non-streaming response even
+    though the client asked to stream (e.g. an agentic tool-use loop that
+    intercepts and returns a complete message) - yielding that dict directly
+    into a `/v1/messages` SSE byte stream would produce a malformed
+    response, so it's synthesized into the message_start/content_block_*/
+    message_delta/message_stop lifecycle a real stream would have sent.
+    """
+    content_blocks: Final = response.get("content") or ()
+    content_events: Final = (
+        event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
+    )
+    # A real message_start always carries a null stop_reason/stop_sequence and
+    # a zero output_tokens - those are only known once generation finishes, so
+    # copying the completed response's final values here would let a client
+    # treat the message as already finished, or double-count output tokens.
+    message_start_usage: Final = {  # mutable-ok: one-shot JSON payload
+        **(response.get("usage") or {}),
+        "output_tokens": 0,
+    }
+    message_start_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_start",
+        "message": {  # mutable-ok: one-shot JSON payload
+            **response,
+            "content": [],  # mutable-ok: one-shot JSON payload
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": message_start_usage,
+        },
+    }
+    message_delta_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_delta",
+        "delta": {  # mutable-ok: one-shot JSON payload
+            "stop_reason": response.get("stop_reason"),
+            "stop_sequence": response.get("stop_sequence"),
+        },
+        "usage": response.get("usage") or {},  # mutable-ok: one-shot JSON payload
+    }
+    return (
+        _sse_event("message_start", message_start_payload),
+        *content_events,
+        _sse_event("message_delta", message_delta_payload),
+        _sse_event("message_stop", {"type": "message_stop"}),  # mutable-ok: one-shot JSON payload
+    )
+
+
+def _anthropic_content_block_events(index: int, block: Mapping[str, object]) -> tuple[bytes, ...]:
+    start_block, deltas = _anthropic_content_block_start_and_deltas(block)
+    start_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_start",
+        "index": index,
+        "content_block": start_block,
+    }
+    stop_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_stop",
+        "index": index,
+    }
+    delta_events: Final = tuple(
+        _sse_event(
+            "content_block_delta",
+            {"type": "content_block_delta", "index": index, "delta": delta},  # mutable-ok: one-shot payload
+        )
+        for delta in deltas
+    )
+    return (
+        _sse_event("content_block_start", start_payload),
+        *delta_events,
+        _sse_event("content_block_stop", stop_payload),
     )
-    return f"event: error\ndata: {payload}\n\n".encode()
 
 
 class AnthropicMessagesStreamHiddenParams(TypedDict):
@@ -97,6 +312,10 @@ class AnthropicMessagesStreamingResponse:
         self.completion_stream = completion_stream
         self._hidden_params = hidden_params
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.completion_stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
         return self
 

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -379,13 +379,19 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     def _translate_legacy_thinking_for_adaptive_model(
         model: str, optional_params: dict, custom_llm_provider: str
     ) -> None:
-        """Translate legacy ``thinking.type=enabled`` to adaptive for 4.6/4.7.
-        Caller-provided ``output_config.effort`` is never overridden.
+        """Translate legacy ``thinking.type=enabled`` to adaptive for the
+        adaptive-thinking models that reject it (4.7+ and the 5 families).
+        Models flagged ``supports_legacy_thinking`` (the 4.6 family) accept the
+        legacy shape natively, so it is forwarded verbatim and the caller's
+        ``budget_tokens`` cap keeps applying. Caller-provided
+        ``output_config.effort`` is never overridden.
         """
         from litellm.llms.anthropic.chat.transformation import AnthropicConfig
 
         if not AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
             return
+        if AnthropicModelInfo._supports_legacy_thinking(model, custom_llm_provider):
+            return
         thinking: Final = optional_params.get("thinking")
         if not isinstance(thinking, dict) or thinking.get("type") != "enabled":
             return

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -152,7 +152,10 @@ class AnthropicResponsesStreamWrapper:
             if block_idx < 0:
                 if not delta:
                     return
-                block_idx = self._open_block(item_id, {"type": "thinking", "thinking": ""})
+                block_idx = self._open_block(
+                    item_id,
+                    {"type": "thinking", "thinking": "", "signature": ""},  # mutable-ok: API message payload
+                )
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",

litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py

@@ -6,12 +6,14 @@ path used for OpenAI and Azure models.
 """
 
 import json
-from collections.abc import Iterable
+from collections.abc import Iterable, Mapping
+from itertools import groupby
 from typing import Any, Final, cast
 
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     TOOL_RESULT_IMAGE_BOUNDARY,
     TOOL_RESULT_IMAGE_PLACEHOLDER,
+    responses_reasoning_item_from_thinking_blocks,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.reasoning_effort_utils import (
@@ -36,7 +38,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
     AnthropicUsage,
 )
-from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
+from litellm.types.llms.openai import (
+    ChatCompletionThinkingBlock,
+    ResponseAPIUsage,
+    ResponsesAPIResponse,
+)
 
 
 class LiteLLMAnthropicToResponsesAPIAdapter:
@@ -81,6 +87,51 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             return source.get("url")
         return None
 
+    @staticmethod
+    def _translate_anthropic_document_block_to_file_part(
+        block: Mapping[str, object],
+    ) -> dict[str, str] | None:  # mutable-ok: API message payload
+        """Convert an Anthropic document block to a Responses input_file part."""
+        raw_source: Final = block.get("source")
+        if not isinstance(raw_source, Mapping):
+            return None
+        source: Final = cast(Mapping[str, object], raw_source)  # cast-ok: untrusted client payload
+        source_type: Final = source.get("type")
+        if source_type == "base64":
+            data: Final = source.get("data")
+            if not isinstance(data, str) or not data:
+                return None
+            raw_media_type: Final = source.get("media_type")
+            media_type: Final = (
+                raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf"
+            )
+            raw_title: Final = block.get("title")
+            filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf"
+            return {  # mutable-ok: API message payload
+                "type": "input_file",
+                "filename": filename,
+                "file_data": f"data:{media_type};base64,{data}",
+            }
+        if source_type == "url":
+            url: Final = source.get("url")
+            if not isinstance(url, str) or not url:
+                return None
+            return {"type": "input_file", "file_url": url}  # mutable-ok: API message payload
+        return None
+
+    @staticmethod
+    def _tool_result_output_value(
+        output_text: str,
+        file_parts: tuple[dict[str, str], ...],  # mutable-ok: json content parts
+    ) -> str | list[dict[str, str]]:  # mutable-ok: API message payload
+        """Plain string output, or a part list when document file parts are present."""
+        if not file_parts:
+            return output_text
+        text_parts: Final = (
+            [{"type": "input_text", "text": output_text}] if output_text else []  # mutable-ok: API message payload
+        )
+        return [*text_parts, *file_parts]  # mutable-ok: API message payload
+
     @staticmethod
     def _translate_midturn_system_content_to_responses(
         content: str | Iterable[AnthropicSystemMessageContent],
@@ -100,6 +151,58 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             if isinstance(block, dict) and block.get("type") == "text" and (text := block.get("text"))  # pyright: ignore[reportUnnecessaryIsInstance]  # untrusted client payload
         ]
 
+    @staticmethod
+    def _summary_part_text(part: object) -> str:
+        if isinstance(part, Mapping):
+            mapping: Final = cast(Mapping[str, Any], part)  # cast-ok: summary parts are untyped provider json
+            return str(mapping.get("text") or "")
+        return str(getattr(part, "text", None) or "")
+
+    @classmethod
+    def _thinking_blocks_from_reasoning_item(
+        cls,
+        summary: Iterable[object],
+    ) -> tuple[dict[str, Any], ...]:  # mutable-ok: API message payload
+        """Anthropic thinking blocks for one Responses reasoning item.
+
+        The signature stays empty: only Anthropic can sign a thinking block, and a stand-in
+        value would be replayed as a real one and rejected by every backend that verifies it.
+        """
+        return tuple(
+            AnthropicResponseContentBlockThinking(
+                type="thinking",
+                thinking=text,
+                signature=None,
+            ).model_dump()
+            for part in summary
+            if (text := cls._summary_part_text(part))
+        )
+
+    @staticmethod
+    def _assistant_block_group_key(indexed_block: tuple[int, Mapping[str, Any]]) -> str:
+        """Group a run of consecutive thinking blocks together; keep every other block alone."""
+        index, block = indexed_block
+        return "thinking" if block.get("type") == "thinking" else f"block:{index}"
+
+    @classmethod
+    def _assistant_group_to_input_item(
+        cls, group: tuple[Mapping[str, Any], ...]
+    ) -> dict[str, Any] | None:  # mutable-ok: API message payload
+        first: Final = group[0]
+        btype: Final = first.get("type")
+        if btype == "thinking":
+            blocks: Final = cast(tuple[ChatCompletionThinkingBlock, ...], group)  # cast-ok: untrusted client payload
+            reasoning_item: Final = responses_reasoning_item_from_thinking_blocks(blocks)
+            return None if reasoning_item is None else dict(reasoning_item)  # mutable-ok: API message payload
+        if btype == "tool_use":
+            return {  # mutable-ok: API message payload
+                "type": "function_call",
+                "call_id": first.get("id", ""),
+                "name": first.get("name", ""),
+                "arguments": json.dumps(first.get("input", {})),  # mutable-ok: API message payload
+            }
+        return None
+
     def translate_messages_to_responses_input(
         self,
         messages: list[AllAnthropicPassThroughMessageValues],
@@ -111,8 +214,10 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
           system text        -> message(role=system, input_text)
           user text          -> message(role=user, input_text)
           user image         -> message(role=user, input_image)
+          user document      -> message(role=user, input_file)
           user tool_result   -> function_call_output
           assistant text     -> message(role=assistant, output_text)
+          assistant thinking -> reasoning
           assistant tool_use -> function_call
         """
         input_items: Final[list[dict[str, Any]]] = []
@@ -164,9 +269,25 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                         {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
                                     )
                                 )
+                        elif btype == "document":
+                            file_part = self._translate_anthropic_document_block_to_file_part(block)
+                            if file_part:
+                                user_parts.append(
+                                    with_prompt_cache_breakpoint(file_part, block.get("prompt_cache_breakpoint"))
+                                )
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
+                            document_candidates = (
+                                tuple(
+                                    self._translate_anthropic_document_block_to_file_part(c)
+                                    for c in inner
+                                    if isinstance(c, dict) and c.get("type") == "document"
+                                )
+                                if isinstance(inner, list)
+                                else ()
+                            )
+                            tool_file_parts = tuple(part for part in document_candidates if part is not None)
                             if inner is None:
                                 output_text = ""
                             elif isinstance(inner, str):
@@ -199,7 +320,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                 {
                                     "type": "function_call_output",
                                     "call_id": tool_use_id,
-                                    "output": output_text,
+                                    "output": self._tool_result_output_value(output_text, tool_file_parts),
                                 }
                             )
                     if tool_image_parts:
@@ -233,27 +354,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         }
                     )
                 elif isinstance(content, list):
-                    asst_parts: list[dict[str, Any]] = []
-                    for block in content:
-                        if not isinstance(block, dict):
-                            continue
-                        btype = block.get("type")
-                        if btype == "text":
-                            asst_parts.append({"type": "output_text", "text": block.get("text", "")})
-                        elif btype == "tool_use":
-                            # tool_use becomes a top-level function_call item
-                            input_items.append(
-                                {
-                                    "type": "function_call",
-                                    "call_id": block.get("id", ""),
-                                    "name": block.get("name", ""),
-                                    "arguments": json.dumps(block.get("input", {})),
-                                }
-                            )
-                        elif btype == "thinking":
-                            thinking_text = block.get("thinking", "")
-                            if thinking_text:
-                                asst_parts.append({"type": "output_text", "text": thinking_text})
+                    blocks = tuple(block for block in content if isinstance(block, dict))
+                    input_items.extend(
+                        item
+                        for _, group in groupby(enumerate(blocks), key=self._assistant_block_group_key)
+                        if (item := self._assistant_group_to_input_item(tuple(block for _, block in group))) is not None
+                    )
+                    asst_parts: list[dict[str, Any]] = [  # mutable-ok: API message payload
+                        {"type": "output_text", "text": block.get("text", "")}  # mutable-ok: API message payload
+                        for block in blocks
+                        if block.get("type") == "text"
+                    ]
                     if asst_parts:
                         input_items.append(
                             {
@@ -514,16 +625,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
 
         for item in response.output:
             if isinstance(item, ResponseReasoningItem):
-                for summary in item.summary:
-                    text = getattr(summary, "text", "")
-                    if text:
-                        content.append(
-                            AnthropicResponseContentBlockThinking(
-                                type="thinking",
-                                thinking=text,
-                                signature=None,
-                            ).model_dump()
-                        )
+                content.extend(self._thinking_blocks_from_reasoning_item(item.summary))
 
             elif isinstance(item, ResponseOutputMessage):
                 for part in item.content:
@@ -555,6 +657,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             content.append(
                                 AnthropicResponseContentBlockText(type="text", text=part.get("text", "")).model_dump()
                             )
+                elif item_type == "reasoning":
+                    content.extend(
+                        self._thinking_blocks_from_reasoning_item(
+                            cast(Iterable[object], item.get("summary") or ()),  # cast-ok: untyped provider json
+                        )
+                    )
                 elif item_type == "function_call":
                     try:
                         input_data = json.loads(item.get("arguments", "{}"))

litellm/llms/anthropic/files/handler.py

@@ -22,19 +22,7 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import CallTypes, LlmProviders, ModelResponse
 
 from ..chat.transformation import AnthropicConfig
-from ..common_utils import AnthropicModelInfo
-
-# Map Anthropic error types to HTTP status codes
-ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = {
-    "invalid_request_error": 400,
-    "authentication_error": 401,
-    "permission_error": 403,
-    "not_found_error": 404,
-    "rate_limit_error": 429,
-    "api_error": 500,
-    "overloaded_error": 503,
-    "timeout_error": 504,
-}
+from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo
 
 
 class AnthropicFilesHandler:

litellm/llms/azure/common_utils.py

@@ -3,6 +3,7 @@ import hashlib
 import json
 import os
 from collections.abc import Callable, Mapping
+from functools import lru_cache
 from typing import Any, Final, Literal, NamedTuple, cast
 
 import httpx
@@ -75,6 +76,24 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
     return {**llm_response_headers, **openai_headers}
 
 
+@lru_cache(maxsize=128)
+def _cached_entra_id_token_provider(
+    tenant_id: str,
+    client_id: str,
+    client_secret: str,
+    scope: str,
+) -> Callable[[], str]:
+    """Build (once per credential set) a bearer token provider backed by a `ClientSecretCredential`.
+
+    The credential caches the access token internally and only talks to Entra ID when it is close
+    to expiry, so reusing the provider keeps one AAD round trip per token lifetime instead of one
+    per request.
+    """
+    from azure.identity import ClientSecretCredential, get_bearer_token_provider
+
+    return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
+
+
 def get_azure_ad_token_from_entra_id(
     tenant_id: str,
     client_id: str,
@@ -93,8 +112,6 @@ def get_azure_ad_token_from_entra_id(
     Returns:
         callable that returns a bearer token.
     """
-    from azure.identity import ClientSecretCredential, get_bearer_token_provider
-
     verbose_logger.debug("Getting Azure AD Token from Entra ID")
 
     if tenant_id.startswith("os.environ/"):
@@ -120,9 +137,13 @@ def get_azure_ad_token_from_entra_id(
     )
     if _tenant_id is None or _client_id is None or _client_secret is None:
         raise ValueError("tenant_id, client_id, and client_secret must be provided")
-    credential: Final = ClientSecretCredential(_tenant_id, _client_id, _client_secret)
 
-    token_provider: Final = get_bearer_token_provider(credential, scope)
+    token_provider: Final = _cached_entra_id_token_provider(
+        tenant_id=_tenant_id,
+        client_id=_client_id,
+        client_secret=_client_secret,
+        scope=scope,
+    )
 
     verbose_logger.debug("token_provider %s", token_provider)
 

litellm/llms/azure/search/__init__.py

@@ -0,0 +1,3 @@
+from litellm.llms.azure.search.transformation import BingGroundingSearchConfig
+
+__all__ = ("BingGroundingSearchConfig",)

litellm/llms/azure/search/transformation.py

@@ -0,0 +1,442 @@
+"""
+Calls the Microsoft Foundry Responses API with the `bing_grounding` or `web_search`
+tool to search the web (Grounding with Bing Search).
+
+Microsoft docs: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding
+
+Setup:
+    1. Set BING_GROUNDING_PROJECT_ENDPOINT to the Foundry project endpoint, e.g.
+       https://<account>.services.ai.azure.com/api/projects/<project>
+    2. Set BING_GROUNDING_MODEL to a model deployment in that project (e.g. gpt-4.1);
+       it runs the grounded search and its tokens are billed on that deployment
+    3. Optional: set BING_GROUNDING_CONNECTION_ID to a Grounding with Bing Search
+       project connection id to use the `bing_grounding` tool; without it the
+       project's built-in `web_search` tool is used
+    4. Auth: pass api_key (an Azure API key, sent in the api-key header), or set
+       BING_GROUNDING_TOKEN to an Entra bearer token for scope
+       https://ai.azure.com/.default, or configure azure-identity (AZURE_CLIENT_ID /
+       AZURE_CLIENT_SECRET / AZURE_TENANT_ID, managed identity, or any
+       DefaultAzureCredential source) and the token is minted automatically
+
+Usage:
+    response = litellm.search(
+        query="latest AI developments",
+        search_provider="bing_grounding",
+        max_results=5,
+    )
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Literal
+
+import httpx
+from pydantic import BaseModel, ConfigDict, ValidationError
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_DOCS_URL: Final = "https://learn.microsoft.com/en-us/azure/ai-foundry/agents/how-to/tools/bing-grounding"
+
+PROJECT_ENDPOINT_ENV: Final = "BING_GROUNDING_PROJECT_ENDPOINT"
+MODEL_ENV: Final = "BING_GROUNDING_MODEL"
+CONNECTION_ID_ENV: Final = "BING_GROUNDING_CONNECTION_ID"
+TOKEN_ENV: Final = "BING_GROUNDING_TOKEN"
+
+ENTRA_SCOPE: Final = "https://ai.azure.com/.default"
+
+_RESPONSES_PATH: Final = "/openai/v1/responses"
+_SNIPPET_FALLBACK_LENGTH: Final = 300
+_UPSTREAM_ERROR_STATUS: Final = 502
+_RESPONSE_COST_HEADER: Final = "llm_provider-x-litellm-response-cost"
+
+
+class _Annotation(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    url: str | None = None
+    title: str | None = None
+    start_index: int | None = None
+    end_index: int | None = None
+
+
+class _ContentPart(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    text: str = ""
+    annotations: tuple[_Annotation, ...] = ()
+
+
+class _OutputItem(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    type: str = ""
+    content: tuple[_ContentPart, ...] = ()
+
+
+class _ErrorBody(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    message: str | None = None
+
+
+class _IncompleteDetails(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    reason: str | None = None
+
+
+class _ResponsesEnvelope(BaseModel):
+    """A Foundry Responses API body. `output` is required: a body without it is not a
+    Responses API response and must not be reported as a successful empty search.
+
+    A 200 body can still carry `status` `failed` or `incomplete`; those are surfaced as
+    errors rather than reported as a successful empty search."""
+
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    output: tuple[_OutputItem, ...]
+    status: str | None = None
+    error: _ErrorBody | None = None
+    incomplete_details: _IncompleteDetails | None = None
+
+
+class _ErrorEnvelope(BaseModel):
+    model_config = ConfigDict(extra="ignore", frozen=True)
+
+    error: _ErrorBody | None = None
+
+
+def _unwrap_error_detail(error_message: str) -> str:
+    """
+    Surface the human-readable message inside Foundry's error envelope.
+
+    Tool failures nest a second JSON document as a string inside `error.message`
+    (observed live for `bing_grounding` connection errors), so the unwrap runs twice.
+    Falls back to the raw body for anything else.
+    """
+    try:
+        envelope: Final = _ErrorEnvelope.model_validate_json(error_message)
+    except ValidationError:
+        return error_message
+    message: Final = envelope.error.message if envelope.error else None
+    if message is None:
+        return error_message
+    try:
+        nested: Final = _ErrorBody.model_validate_json(message)
+    except ValidationError:
+        return message
+    return nested.message or message
+
+
+def _snippet(text: str, annotation: _Annotation) -> str:
+    """
+    The text a citation supports, not the citation marker itself.
+
+    A url_citation's start/end indices span the inline marker ("([host](url))"),
+    which follows the claim it backs, so the snippet is the marker's own line up
+    to where the marker starts.
+    """
+    start: Final = annotation.start_index
+    marker_start: Final = start if start is not None and 0 <= start <= len(text) else len(text)
+    claim: Final = text[:marker_start].rsplit("\n", 1)[-1].strip()
+    if claim:
+        return claim[-_SNIPPET_FALLBACK_LENGTH:]
+    return text[:_SNIPPET_FALLBACK_LENGTH]
+
+
+def _citation_results(envelope: _ResponsesEnvelope) -> tuple[SearchResult, ...]:
+    """One result per cited URL: first occurrence wins, order preserved as answered."""
+    cited: Final = tuple(
+        SearchResult(
+            title=annotation.title or "",
+            url=annotation.url or "",
+            snippet=_snippet(part.text, annotation),
+            date=None,
+            last_updated=None,
+        )
+        for item in envelope.output
+        if item.type == "message"
+        for part in item.content
+        if part.type == "output_text"
+        for annotation in part.annotations
+        if annotation.type == "url_citation" and annotation.url
+    )
+    first_by_url: Final = MappingProxyType({result.url: result for result in reversed(cited)})
+    return tuple(first_by_url[url] for url in dict.fromkeys(result.url for result in cited))
+
+
+def _valid_max_results(max_results: object) -> int | None:
+    """A positive-int `max_results`, else None. Rejects bools, an `int` subclass, and
+    non-positive values so neither the request-side `count` nor the response-side cap
+    forwards a value the other would silently ignore.
+    """
+    if isinstance(max_results, bool) or not isinstance(max_results, int):
+        return None
+    return max_results if max_results > 0 else None
+
+
+def _requested_max_results(response_kwargs: Mapping[str, object]) -> int | None:
+    """The unified `max_results` cap the caller asked for, if any.
+
+    The built-in web_search tool has no server-side result-count knob, so the cap is
+    enforced here after the fact; connection mode also honors it as a hard ceiling on
+    top of the tool's `count` hint.
+    """
+    optional_params: Final = response_kwargs.get("optional_params")
+    if not isinstance(optional_params, Mapping):
+        return None
+    return _valid_max_results(optional_params.get("max_results"))
+
+
+def _capped(results: tuple[SearchResult, ...], max_results: int | None) -> tuple[SearchResult, ...]:
+    return results[:max_results] if max_results is not None else results
+
+
+class _SearchConfiguration(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    project_connection_id: str
+    count: int | None = None
+
+
+class _BingGroundingParams(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    search_configurations: tuple[_SearchConfiguration, ...]
+
+
+class _BingGroundingTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["bing_grounding"] = "bing_grounding"
+    bing_grounding: _BingGroundingParams
+
+
+class _UserLocation(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["approximate"] = "approximate"
+    country: str
+
+
+class _WebSearchTool(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: Literal["web_search"] = "web_search"
+    user_location: _UserLocation | None = None
+
+
+class _ResponsesRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+    input: str
+    tools: tuple[_BingGroundingTool | _WebSearchTool, ...]
+
+
+def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool | _WebSearchTool:
+    connection_id: Final = get_secret_str(CONNECTION_ID_ENV)
+    max_results: Final = optional_params.get("max_results")
+    country: Final = optional_params.get("country")
+    if connection_id:
+        configuration: Final = _SearchConfiguration(
+            project_connection_id=connection_id,
+            count=_valid_max_results(max_results),
+        )
+        return _BingGroundingTool(bing_grounding=_BingGroundingParams(search_configurations=(configuration,)))
+    location: Final = _UserLocation(country=country.upper()) if isinstance(country, str) else None
+    return _WebSearchTool(user_location=location)
+
+
+def _default_entra_token_minter() -> str:
+    from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
+
+    return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
+
+
+class BingGroundingSearchConfig(BaseSearchConfig):
+    def __init__(self, entra_token_minter: Callable[[], str] | None = None) -> None:
+        super().__init__()
+        self._entra_token_minter = entra_token_minter
+
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "Grounding with Bing Search"
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.validate_environment signature
+        api_key: str | None = None,
+        api_base: str | None = None,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: the http handler passes this straight to httpx as headers
+        """
+        Validate environment and return headers.
+
+        Returns a new dict rather than mutating ``headers``: the http handler calls this
+        a second time after ``litellm/search/main.py`` already did, so it has to be idempotent.
+        """
+        return {  # mutable-ok: httpx requires a plain dict of headers
+            **headers,
+            **self._auth_header(api_key, api_base),
+            "Content-Type": "application/json",
+        }
+
+    def _auth_header(self, api_key: str | None, api_base: str | None) -> Mapping[str, str]:
+        """
+        A caller-supplied ``api_key`` is an Azure API key and rides the ``api-key`` header;
+        an Entra bearer token (``BING_GROUNDING_TOKEN`` or one minted via azure-identity)
+        rides ``Authorization: Bearer``. Foundry rejects the wrong scheme for each.
+        """
+        if api_key:
+            return MappingProxyType({"api-key": api_key})
+        token: Final = self.resolve_server_api_key(
+            caller_api_key=None,
+            caller_api_base=api_base,
+            key_env_vars=(TOKEN_ENV,),
+            base_env_var=PROJECT_ENDPOINT_ENV,
+            default_api_base=None,
+        ) or self._mint_entra_token(api_base)
+        return MappingProxyType({"Authorization": f"Bearer {token}"})
+
+    def _mint_entra_token(self, caller_api_base: str | None) -> str:
+        self._assert_trusted_api_base_for_server_credential(
+            caller_api_base, None, PROJECT_ENDPOINT_ENV, "Azure AD token"
+        )
+        minter: Final = self._entra_token_minter or _default_entra_token_minter
+        try:
+            return minter()
+        except Exception as e:
+            raise ValueError(
+                f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
+                f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "
+                f"AZURE_CLIENT_SECRET / AZURE_TENANT_ID or any DefaultAzureCredential source) "
+                f"for scope {ENTRA_SCOPE}. Underlying error: {e}"
+            ) from e
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        optional_params: dict[str, object],  # mutable-ok: BaseSearchConfig.get_complete_url signature
+        data: dict[str, object] | list[dict[str, object]] | None = None,  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.get_complete_url signature
+    ) -> str:
+        resolved_base: Final = api_base or get_secret_str(PROJECT_ENDPOINT_ENV)
+        if not resolved_base:
+            raise ValueError(
+                f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project "
+                f"endpoint, e.g. https://<account>.services.ai.azure.com/api/projects/<project>."
+            )
+        trimmed: Final = resolved_base.rstrip("/")
+        if trimmed.endswith(_RESPONSES_PATH):
+            return trimmed
+        return f"{trimmed}{_RESPONSES_PATH}"
+
+    def transform_search_request(
+        self,
+        query: str | list[str],  # mutable-ok: BaseSearchConfig.transform_search_request signature
+        optional_params: dict[str, object],  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_request signature
+    ) -> dict[str, object]:  # mutable-ok: the http handler passes this straight to httpx as the JSON body
+        """
+        Transform Search request to the Foundry Responses API format.
+
+        The unified params map as far as the API allows:
+        - max_results -> the bing_grounding search configuration's `count`; the built-in
+          web_search tool has no result-count knob, so that mode instead caps the returned
+          results after the fact (see transform_search_response)
+        - country -> web_search's approximate `user_location` (bing_grounding's `market`
+          wants a full locale like en-US, which a bare country code cannot fill)
+        - search_domain_filter, max_tokens_per_page -> no API equivalent, dropped
+        """
+        model: Final = get_secret_str(MODEL_ENV)
+        if not model:
+            raise ValueError(
+                f"{MODEL_ENV} is not set. Set it to a model deployment in the Foundry project "
+                f"that runs the grounded search, e.g. gpt-4.1."
+            )
+        request: Final = _ResponsesRequest(
+            model=model,
+            input=" ".join(query) if isinstance(query, list) else query,
+            tools=(_search_tool(optional_params),),
+        )
+        return request.model_dump(mode="json", exclude_none=True)
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_response signature
+    ) -> SearchResponse:
+        try:
+            parsed: Final = _ResponsesEnvelope.model_validate_json(raw_response.content)
+        except ValidationError as e:
+            raise self.get_error_class(
+                error_message=f"response does not match the Foundry Responses API schema: {e}",
+                status_code=raw_response.status_code,
+                headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+            )
+        if parsed.status == "failed":
+            detail: Final = (
+                parsed.error.message if parsed.error and parsed.error.message else "the grounded search failed"
+            )
+            raise self._upstream_error(detail, raw_response)
+        results: Final = _capped(_citation_results(parsed), _requested_max_results(kwargs))
+        if not results and parsed.status == "incomplete":
+            reason: Final = (
+                parsed.incomplete_details.reason
+                if parsed.incomplete_details and parsed.incomplete_details.reason
+                else "unknown reason"
+            )
+            raise self._upstream_error(f"the grounded search was incomplete: {reason}", raw_response)
+        return self._priced(results)
+
+    def _upstream_error(self, detail: str, raw_response: httpx.Response) -> Exception:
+        return self.get_error_class(
+            error_message=detail,
+            status_code=_UPSTREAM_ERROR_STATUS,
+            headers=dict(raw_response.headers),  # mutable-ok: BaseSearchConfig.get_error_class signature
+        )
+
+    def _priced(self, results: tuple[SearchResult, ...]) -> SearchResponse:
+        """web_search mode runs no paid Grounding with Bing transaction, so it must not
+        inherit the connection-mode ``bing_grounding/search`` price; zero its per-query
+        cost while leaving connection mode to the cost map."""
+        response: Final = SearchResponse(
+            results=list(results),  # mutable-ok: SearchResponse.results is list[SearchResult]
+            object="search",
+        )
+        if get_secret_str(CONNECTION_ID_ENV):
+            return response
+        response._hidden_params[
+            "additional_headers"
+        ] = {  # mutable-ok: response_cost_calculator writes into _hidden_params
+            _RESPONSE_COST_HEADER: 0.0
+        }
+        return response
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.get_error_class signature
+    ) -> Exception:
+        detail: Final = _unwrap_error_detail(error_message).rstrip(". ")
+        return BaseLLMException(
+            status_code=status_code,
+            message=f"Grounding with Bing Search: {detail}. See {_DOCS_URL} for details.",
+            headers=headers,
+        )

litellm/llms/azure_ai/azure_model_router/transformation.py

@@ -65,15 +65,24 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
 
         Extracts the actual model used from the Azure response (e.g., gpt-5-nano-2025-08-07)
         and returns it with the azure_ai/ prefix for proper display and cost tracking.
+
+        Also stamps that model onto ``_hidden_params`` so downstream consumers (spend logs,
+        response restamping) can read it instead of guessing the route from the model string.
         """
-        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+        from litellm.llms.azure_ai.common_utils import (
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+            AzureFoundryModelInfo,
+        )
+        from litellm.router_utils.add_retry_fallback_headers import (
+            get_hidden_params_dict,
+        )
 
         # Get base model for the parent call (strips routing prefixes for API compatibility)
         base_model: Final[str] = AzureFoundryModelInfo.get_base_model(model)
 
         # Call parent transform_response first - this will extract the actual model
         # from the raw response (e.g., "gpt-5-nano-2025-08-07")
-        model_response = super().transform_response(
+        transformed_response: Final = super().transform_response(
             model=base_model,
             raw_response=raw_response,
             model_response=model_response,
@@ -86,7 +95,15 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
-        return model_response
+        selected_model: Final = transformed_response.model
+        if selected_model:
+            # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a
+            # class-level dict, so an in-place write can bleed into unrelated responses.
+            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter  # mutable-ok: ModelResponse requires _hidden_params to be a plain dict
+                **get_hidden_params_dict(transformed_response),
+                AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model,
+            }
+        return transformed_response
 
     def calculate_additional_costs(self, model: str, prompt_tokens: int, completion_tokens: int) -> dict | None:
         """

litellm/llms/azure_ai/chat/transformation.py

@@ -30,7 +30,12 @@ class AzureFoundryErrorStrings(str, enum.Enum):
     SET_EXTRA_PARAMETERS_TO_PASS_THROUGH = "Set extra-parameters to 'pass-through'"
 
 
-NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = ("thinking_blocks", "provider_specific_fields", "cache_control")
+NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
+    "thinking_blocks",
+    "reasoning_content",
+    "provider_specific_fields",
+    "cache_control",
+)
 
 
 class AzureAIStudioConfig(OpenAIConfig):
@@ -173,7 +178,8 @@ class AzureAIStudioConfig(OpenAIConfig):
         """
         - Azure AI Studio doesn't support content as a list. This handles:
             1. Strips message fields that are not part of the OpenAI chat-completions
-               schema (thinking_blocks, provider_specific_fields, cache_control).
+               schema (thinking_blocks, reasoning_content, provider_specific_fields,
+               cache_control).
                Azure AI Foundry backends set additionalProperties=false and reject
                these with "Extra inputs are not permitted", which breaks multi-turn
                Anthropic-format clients that echo thinking blocks back as history.

litellm/llms/azure_ai/common_utils.py

@@ -1,9 +1,57 @@
+from collections.abc import Mapping
 from typing import Final, Literal
 
 import litellm
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
+from litellm.types.router import GenericLiteLLMParams
+
+AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
+
+
+def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
+    """
+    Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
+
+    Accepts the same credential set as the `azure` provider: service principal
+    (`tenant_id` / `client_id` / `client_secret`), a pre-fetched `azure_ad_token`, an OIDC
+    federated token, username/password, or `DefaultAzureCredential` / managed identity.
+    """
+    from litellm.llms.azure.common_utils import get_azure_ad_token
+
+    params = GenericLiteLLMParams.model_validate(litellm_params) if litellm_params else GenericLiteLLMParams()
+
+    return get_azure_ad_token(params)
+
+
+def get_azure_ai_auth_headers(
+    api_key: str | None,
+    litellm_params: Mapping[str, object] | None = None,
+    api_key_header: AzureAIApiKeyHeader = "Authorization",
+    api_key_env_var: str = "AZURE_AI_API_KEY",
+) -> Mapping[str, str]:
+    """
+    Build the auth headers for an Azure AI Foundry route.
+
+    Prefers the API key when one is configured, and otherwise falls back to Entra ID / OAuth,
+    sending the access token as a bearer token.
+    """
+    if api_key:
+        return {api_key_header: f"Bearer {api_key}" if api_key_header == "Authorization" else api_key}
+
+    azure_ad_token = get_azure_ai_entra_token(litellm_params=litellm_params)
+    if azure_ad_token:
+        return {"Authorization": f"Bearer {azure_ad_token}"}
+
+    raise ValueError(
+        f"Missing Azure AI credentials - set an API key (`api_key` or {api_key_env_var}), or Entra ID / OAuth "
+        "credentials (`tenant_id` + `client_id` + `client_secret`, `azure_ad_token`, an OIDC token, or a managed "
+        "identity with `litellm.enable_azure_ad_token_refresh = True`)"
+    )
+
+
+AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
 
 
 class AzureFoundryModelInfo(BaseLLMModelInfo):
@@ -37,13 +85,48 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
             return "model_router"
         return "default"
 
+    @staticmethod
+    def get_model_router_selected_model(hidden_params: Mapping[str, object] | None) -> str | None:
+        """The model Azure Model Router actually served, stamped by ``AzureModelRouterConfig``.
+
+        Reading this beats re-deriving the route from a model string: the stamp is set on the
+        code path that was actually taken, so it holds no matter what the caller named the model.
+        """
+        if not hidden_params:
+            return None
+        selected: Final = hidden_params.get(AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY)
+        if isinstance(selected, str) and selected:
+            return selected
+        return None
+
+    @staticmethod
+    def is_model_router_call(
+        model: str | None = None,
+        hidden_params: Mapping[str, object] | None = None,
+    ) -> bool:
+        """Whether a request went down the Azure Model Router route.
+
+        Prefers the response stamp, then the deployment's litellm model path, and only then the
+        caller-supplied name. The last two go through ``get_azure_ai_route`` so the model-router
+        name heuristic lives in exactly one place.
+        """
+        if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
+            return True
+        deployment_model: Final = (
+            hidden_params.get("litellm_model_name") or hidden_params.get("model") if hidden_params is not None else None
+        )
+        return any(
+            isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"
+            for candidate in (deployment_model, model)
+        )
+
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")
 
     @staticmethod
     def get_api_key(api_key: str | None = None) -> str | None:
-        return api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_AI_API_KEY")
+        return api_key or litellm.api_key or get_secret_str("AZURE_AI_API_KEY")
 
     @property
     def api_version(self, api_version: str | None = None) -> str | None:

litellm/llms/azure_ai/image_edit/flux2_transformation.py

@@ -5,7 +5,10 @@ from typing import Any, Final
 from httpx._types import RequestFiles
 
 import litellm
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.azure_ai.image_generation.flux_transformation import (
     AzureFoundryFluxImageGenerationConfig,
 )
@@ -71,16 +74,13 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
         """
         Validate Azure AI Foundry environment and set up authentication
         """
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
-            )
-
         headers.update(
             {
-                "Api-Key": api_key,
+                **get_azure_ai_auth_headers(
+                    api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                    litellm_params=litellm_params,
+                    api_key_header="Api-Key",
+                ),
                 "Content-Type": "application/json",
             }
         )

litellm/llms/azure_ai/image_edit/mai_transformation.py

@@ -3,7 +3,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
 import httpx
 from httpx._types import RequestFiles
 
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.azure_ai.image_generation.mai_transformation import (
     AzureFoundryMAIImageGenerationConfig,
 )
@@ -91,15 +94,13 @@ class AzureFoundryMAIImageEditConfig(OpenAIImageEditConfig):
         litellm_params: dict | None = None,
         api_base: str | None = None,
     ) -> dict:
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. "
-                "Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
+        headers.update(
+            get_azure_ai_auth_headers(
+                api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                litellm_params=litellm_params,
+                api_key_header="api-key",
             )
-
-        headers.update({"api-key": api_key})
+        )
         return headers
 
     def get_complete_url(

litellm/llms/azure_ai/image_edit/transformation.py

@@ -3,7 +3,10 @@ from typing import Final
 import httpx
 
 import litellm
-from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+from litellm.llms.azure_ai.common_utils import (
+    AzureFoundryModelInfo,
+    get_azure_ai_auth_headers,
+)
 from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.utils import _add_path_to_api_base
@@ -30,19 +33,14 @@ class AzureFoundryFluxImageEditConfig(OpenAIImageEditConfig):
     ) -> dict:
         """
         Validate Azure AI Foundry environment and set up authentication
-        Uses Api-Key header format
+        Uses the Api-Key header format, or an Entra ID / OAuth bearer token when no key is set
         """
-        api_key = AzureFoundryModelInfo.get_api_key(api_key)
-
-        if not api_key:
-            raise ValueError(
-                f"Azure AI API key is required for model {model}. Set AZURE_AI_API_KEY environment variable or pass api_key parameter."
-            )
-
         headers.update(
-            {
-                "Api-Key": api_key,  # Azure AI Foundry uses Api-Key header format
-            }
+            get_azure_ai_auth_headers(
+                api_key=AzureFoundryModelInfo.get_api_key(api_key),
+                litellm_params=litellm_params,
+                api_key_header="Api-Key",
+            )
         )
         return headers
 

litellm/llms/azure_ai/ocr/document_intelligence/transformation.py

@@ -26,6 +26,7 @@ from litellm.constants import (
 )
 from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin, encode_url_path_segment
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.base_llm.ocr.transformation import (
     OCR_REQUEST_FORMAT_PARAM,
     BaseOCRConfig,
@@ -236,17 +237,13 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         """
         Validate environment and return headers for Azure Document Intelligence.
 
-        Authentication uses Ocp-Apim-Subscription-Key header.
+        Authentication uses the Ocp-Apim-Subscription-Key header, or an Entra ID / OAuth bearer
+        token when no subscription key is set.
         """
         # Get API key from environment if not provided
         if api_key is None:
             api_key = get_secret_str(AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR)
 
-        if api_key is None:
-            raise ValueError(
-                "Missing Azure Document Intelligence API Key - Set AZURE_DOCUMENT_INTELLIGENCE_API_KEY environment variable or pass api_key parameter"
-            )
-
         # Validate API base/endpoint is provided
         if api_base is None:
             api_base = get_secret_str("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT")
@@ -257,7 +254,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
             )
 
         headers = {
-            "Ocp-Apim-Subscription-Key": api_key,
+            **get_azure_ai_auth_headers(
+                api_key=api_key,
+                litellm_params=litellm_params,
+                api_key_header="Ocp-Apim-Subscription-Key",
+                api_key_env_var=AZURE_DOCUMENT_INTELLIGENCE_API_KEY_ENV_VAR,
+            ),
             "Content-Type": "application/json",
             **headers,
         }

litellm/llms/azure_ai/ocr/transformation.py

@@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     async_convert_url_to_base64,
     convert_url_to_base64,
 )
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.base_llm.ocr.transformation import DocumentType, OCRRequestData
 from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
 from litellm.secret_managers.main import get_secret_str
@@ -47,17 +48,12 @@ class AzureAIOCRConfig(MistralOCRConfig):
         """
         Validate environment and return headers for Azure AI OCR.
 
-        Azure AI uses Bearer token authentication with AZURE_AI_API_KEY.
+        Authenticates with AZURE_AI_API_KEY, or with an Entra ID / OAuth token when no key is set.
         """
         # Get API key from environment if not provided
         if api_key is None:
             api_key = get_secret_str(AZURE_AI_OCR_API_KEY_ENV_VAR)
 
-        if api_key is None:
-            raise ValueError(
-                "Missing Azure AI API Key - A call is being made to Azure AI but no key is set either in the environment variables or via params"
-            )
-
         # Validate API base is provided
         if api_base is None:
             api_base = get_secret_str("AZURE_AI_API_BASE")
@@ -68,7 +64,7 @@ class AzureAIOCRConfig(MistralOCRConfig):
             )
 
         headers = {
-            "Authorization": f"Bearer {api_key}",
+            **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
             "Content-Type": "application/json",
             **headers,
         }

litellm/llms/azure_ai/rerank/transformation.py

@@ -2,12 +2,14 @@
 Translate between Cohere's `/rerank` format and Azure AI's `/rerank` format.
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
 
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
 from litellm.llms.cohere.rerank.transformation import CohereRerankConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import RerankResponse
@@ -64,15 +66,13 @@ class AzureAIRerankConfig(CohereRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("AZURE_AI_API_KEY") or litellm.azure_key
 
-        if api_key is None:
-            raise ValueError("Azure AI API key is required. Please set 'AZURE_AI_API_KEY' or 'litellm.azure_key'")
-
         default_headers: Final = {
-            "Authorization": f"Bearer {api_key}",
+            **get_azure_ai_auth_headers(api_key=api_key, litellm_params=litellm_params),
             "accept": "application/json",
             "content-type": "application/json",
         }

litellm/llms/base_llm/rerank/transformation.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -24,6 +25,7 @@ class BaseRerankConfig(ABC):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         pass
 

litellm/llms/base_llm/videos/transformation.py

@@ -1,9 +1,10 @@
 import types
 from abc import ABC, abstractmethod
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any
 
 import httpx
-from httpx._types import RequestFiles
+from httpx._types import FileContent, RequestFiles
 
 from litellm.types.responses.main import *
 from litellm.types.router import GenericLiteLLMParams
@@ -91,6 +92,14 @@ class BaseVideoConfig(ABC):
             raise ValueError("api_base is required")
         return api_base
 
+    def use_multipart_form_data(self) -> bool:
+        """
+        Whether video create requests without files must still be sent as
+        multipart/form-data (the encoding the OpenAI SDK always uses for
+        /videos), instead of falling back to JSON.
+        """
+        return False
+
     @abstractmethod
     def transform_video_create_request(
         self,
@@ -332,14 +341,18 @@ class BaseVideoConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
+        video_file: FileContent | None = None,
         extra_body: dict[str, Any] | None = None,
         prefetched_source_data: dict[str, Any] | None = None,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
         """
-        Transform the video edit request into a URL and JSON data.
+        Transform the video edit request into a URL plus either JSON data or
+        multipart form fields and files.
 
         Returns:
-            Tuple[str, Dict]: (url, data) for the POST request
+            tuple[str, Mapping[str, object], RequestFiles | None]: (url, data,
+            files). When files is None the handler sends data as JSON; otherwise
+            data holds the form fields and files holds the uploaded source video.
         """
         raise NotImplementedError("video edit is not supported for this provider")
 

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1617,6 +1617,8 @@ class AmazonConverseConfig(BaseConfig):
         }
         if additional_request_params:
             data["additionalModelRequestFields"] = additional_request_params
+            if "thinking" in additional_request_params:
+                data["additionalModelResponseFieldPaths"] = ("/usage/output_tokens_details",)
         if system_content_blocks:
             data["system"] = system_content_blocks
 
@@ -1801,6 +1803,17 @@ class AmazonConverseConfig(BaseConfig):
                 thinking_blocks_list.append(_redacted_block)
         return thinking_blocks_list
 
+    @staticmethod
+    def thinking_tokens_from_additional_fields(additional_fields: object) -> int | None:
+        """Converse omits thinking tokens from its usage block; they only arrive under
+        ``additionalModelResponseFields`` when ``/usage/output_tokens_details`` is requested."""
+        if not isinstance(additional_fields, Mapping):
+            return None
+        usage: Final = additional_fields.get("usage")
+        if not isinstance(usage, Mapping):
+            return None
+        return AnthropicConfig.thinking_tokens_from_usage(usage)
+
     @staticmethod
     def is_converse_usage_shape(usage_object: Mapping[str, object]) -> bool:
         """Converse-family models report camelCase token counts, not Anthropic's snake_case."""
@@ -1842,6 +1855,7 @@ class AmazonConverseConfig(BaseConfig):
         usage: ConverseTokenUsageBlock,
         reasoning_content: str | None = None,
         thinking_ran: bool = False,
+        provider_reasoning_tokens: int | None = None,
     ) -> Usage:
         input_tokens = usage["inputTokens"]
         output_tokens: Final = usage["outputTokens"]
@@ -1862,9 +1876,14 @@ class AmazonConverseConfig(BaseConfig):
             cache_creation_tokens=cache_creation_input_tokens,
             text_tokens=raw_input_tokens,
         )
-        reasoning_tokens: Final = (
+        estimated_reasoning_tokens: Final = (
             token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0
         )
+        reasoning_tokens: Final = (
+            min(max(0, provider_reasoning_tokens), output_tokens)
+            if provider_reasoning_tokens is not None
+            else estimated_reasoning_tokens
+        )
         completion_tokens_details: Final = (
             CompletionTokensDetailsWrapper(
                 reasoning_tokens=reasoning_tokens,
@@ -2272,6 +2291,9 @@ class AmazonConverseConfig(BaseConfig):
             completion_response["usage"],
             reasoning_content=chat_completion_message.get("reasoning_content"),
             thinking_ran=reasoningContentBlocks is not None,
+            provider_reasoning_tokens=self.thinking_tokens_from_additional_fields(
+                completion_response.get("additionalModelResponseFields")
+            ),
         )
 
         ## HANDLE TOOL CALLS

litellm/llms/bedrock/chat/invoke_handler.py

@@ -331,6 +331,7 @@ class AWSEventStreamDecoder:
         self.json_mode = json_mode
         self._current_tool_name: str | None = None
         self._thinking_ran = False
+        self._provider_reasoning_tokens: int | None = None
 
     def check_empty_tool_call_args(self) -> bool:
         """
@@ -559,14 +560,22 @@ class AWSEventStreamDecoder:
                 tool_use = self._handle_converse_stop_event(content_block_index)
             elif "stopReason" in chunk_data:
                 finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
+                self._provider_reasoning_tokens = AmazonConverseConfig.thinking_tokens_from_additional_fields(
+                    chunk_data.get("additionalModelResponseFields")
+                )
             elif "usage" in chunk_data:
                 usage = converse_config.transform_usage(
                     chunk_data.get("usage", {}),
                     thinking_ran=self._thinking_ran,
+                    provider_reasoning_tokens=self._provider_reasoning_tokens,
                 )
             if thinking_blocks:
                 self._thinking_ran = True
 
+            carries_message_content: Final = any(
+                key in chunk_data for key in ("start", "delta", "contentBlockIndex", "stopReason", "trace")
+            )
+
             model_response_provider_specific_fields: Final = {}
             if "trace" in chunk_data:
                 trace: Final = chunk_data.get("trace")
@@ -577,8 +586,8 @@ class AWSEventStreamDecoder:
                         finish_reason=finish_reason,
                         index=0,  # Always 0 - Bedrock never returns multiple choices
                         delta=Delta(
-                            content=text,
-                            role="assistant",
+                            content=text if carries_message_content else None,
+                            role="assistant" if carries_message_content else None,
                             tool_calls=[tool_use] if tool_use else None,
                             provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
                             thinking_blocks=thinking_blocks,

litellm/llms/bedrock/passthrough/transformation.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Optional, cast
 
 from httpx import Response
@@ -93,6 +94,9 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             endpoint_url,
         )
 
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        return None
+
     def sign_request(
         self,
         headers: dict,
@@ -109,6 +113,7 @@ class BedrockPassthroughConfig(BaseAWSLLM, BedrockModelInfo, BedrockEventStreamD
             request_data=request_data or {},
             api_base=api_base,
             model=model,
+            api_key=self.get_bedrock_bearer_token(optional_params),
         )
 
     def logging_non_streaming_response(

litellm/llms/bedrock/rerank/handler.py

@@ -29,6 +29,7 @@ class BedrockRerankHandler(BaseAWSLLM):
     async def arerank(
         self,
         prepared_request: BedrockPreparedRequest,
+        logging_obj: LitellmLogging,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ):
@@ -40,6 +41,7 @@ class BedrockRerankHandler(BaseAWSLLM):
                 headers=dict(prepared_request["prepped"].headers),
                 data=prepared_request["body"],
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
             response.raise_for_status()
         except httpx.HTTPStatusError as err:
@@ -98,6 +100,7 @@ class BedrockRerankHandler(BaseAWSLLM):
         if _is_async:
             return self.arerank(
                 prepared_request,
+                logging_obj=logging_obj,
                 timeout=timeout,
                 client=client if client is not None and isinstance(client, AsyncHTTPHandler) else None,
             )

litellm/llms/bedrock_mantle/common_utils.py

@@ -13,6 +13,7 @@ global state.
 """
 
 import re
+from collections.abc import Mapping
 from typing import Final
 
 from botocore.exceptions import (
@@ -31,30 +32,39 @@ BEDROCK_MANTLE_DEFAULT_REGION: Final = "us-east-1"
 MANTLE_HOST_RE: Final = re.compile(r"^https?://bedrock-mantle\.([^/.]+)\.api\.aws", re.IGNORECASE)
 
 
+def resolve_mantle_bearer_token(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+
+
+def resolve_mantle_region(params: Mapping[str, object]) -> str:
+    region: Final = params.get("aws_region_name")
+    if isinstance(region, str) and region:
+        BaseAWSLLM._validate_aws_region_name(region)
+        return region
+    api_base: Final = params.get("api_base")
+    base: Final = (api_base if isinstance(api_base, str) else None) or get_secret_str("BEDROCK_MANTLE_API_BASE")
+    if base:
+        match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
+        if match:
+            return match.group(1)
+    return (
+        get_secret_str("BEDROCK_MANTLE_REGION")
+        or get_secret_str("AWS_REGION_NAME")
+        or get_secret_str("AWS_REGION")
+        or BEDROCK_MANTLE_DEFAULT_REGION
+    )
+
+
 class BedrockMantleAuthMixin:
     _aws_signer: BaseAWSLLM
 
     @staticmethod
     def _resolve_bearer_token(api_key: str | None) -> str | None:
-        return api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") or get_secret_str("AWS_BEARER_TOKEN_BEDROCK")
+        return resolve_mantle_bearer_token(api_key)
 
     @staticmethod
     def _resolve_region(params: dict) -> str:
-        region: Final = params.get("aws_region_name")
-        if region:
-            BaseAWSLLM._validate_aws_region_name(region)
-            return region
-        base: Final = params.get("api_base") or get_secret_str("BEDROCK_MANTLE_API_BASE")
-        if base:
-            match: Final = MANTLE_HOST_RE.match(base.rstrip("/"))
-            if match:
-                return match.group(1)
-        return (
-            get_secret_str("BEDROCK_MANTLE_REGION")
-            or get_secret_str("AWS_REGION_NAME")
-            or get_secret_str("AWS_REGION")
-            or BEDROCK_MANTLE_DEFAULT_REGION
-        )
+        return resolve_mantle_region(params)
 
     def sign_request(
         self,

litellm/llms/bedrock_mantle/passthrough/transformation.py

@@ -0,0 +1,71 @@
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, Literal, Optional
+
+from httpx import Response
+
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig
+from litellm.llms.bedrock_mantle.common_utils import (
+    MANTLE_HOST_RE,
+    resolve_mantle_bearer_token,
+    resolve_mantle_region,
+)
+from litellm.types.utils import LlmProviders
+
+if TYPE_CHECKING:
+    from litellm.types.utils import CostResponseTypes
+
+
+class BedrockMantlePassthroughConfig(BedrockPassthroughConfig):
+    """Native Bedrock runtime passthrough (InvokeModel, Converse) for deployments declared as bedrock_mantle.
+
+    The Mantle host only serves the OpenAI-compatible surface, so a Mantle api_base lends its region and the
+    request itself goes to bedrock-runtime, signed with the deployment's Bearer token or SigV4 credentials.
+    """
+
+    def _get_aws_region_name(
+        self,
+        optional_params: Mapping[str, object],
+        model: str | None = None,
+        model_id: str | None = None,
+    ) -> str:
+        return resolve_mantle_region(optional_params)
+
+    def get_runtime_endpoint(
+        self,
+        api_base: str | None,
+        aws_bedrock_runtime_endpoint: str | None,
+        aws_region_name: str,
+        endpoint_type: Literal["runtime", "agent", "agentcore"] | None = "runtime",
+    ) -> tuple[str, str]:
+        is_mantle_host: Final = api_base is not None and MANTLE_HOST_RE.match(api_base.rstrip("/")) is not None
+        return super().get_runtime_endpoint(
+            api_base=None if is_mantle_host else api_base,
+            aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
+            aws_region_name=aws_region_name,
+            endpoint_type=endpoint_type,
+        )
+
+    def get_bedrock_bearer_token(self, litellm_params: Mapping[str, object]) -> str | None:
+        api_key: Final = litellm_params.get("api_key")
+        return resolve_mantle_bearer_token(api_key if isinstance(api_key, str) else None)
+
+    def logging_non_streaming_response(
+        self,
+        model: str,
+        custom_llm_provider: str,
+        httpx_response: Response,
+        request_data: dict,  # mutable-ok: mirrors the inherited BedrockPassthroughConfig signature
+        logging_obj: Logging,
+        endpoint: str,
+    ) -> Optional["CostResponseTypes"]:
+        is_converse: Final = "invoke" not in endpoint and "converse" in endpoint
+        shape_provider: Final = LlmProviders.BEDROCK.value if is_converse else custom_llm_provider
+        return super().logging_non_streaming_response(
+            model=model,
+            custom_llm_provider=shape_provider,
+            httpx_response=httpx_response,
+            request_data=request_data,
+            logging_obj=logging_obj,
+            endpoint=endpoint,
+        )

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -15,8 +15,12 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
+import json
+from collections.abc import Mapping
 from typing import Any, Final
 
+from typing_extensions import ReadOnly, TypedDict
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -50,6 +54,33 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 
 _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
 
+_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
+_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
+_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
+
+
+class _RewrittenOutputTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+
+
+class _RewrittenAssistantMessageItem(TypedDict):
+    type: ReadOnly[str]
+    role: ReadOnly[str]
+    content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
+
+
+class _RewrittenCompactionItem(TypedDict):
+    type: ReadOnly[str]
+    encrypted_content: ReadOnly[str]
+
+
+class _RewrittenFunctionCallItem(TypedDict):
+    type: ReadOnly[str]
+    call_id: ReadOnly[str]
+    name: ReadOnly[str]
+    arguments: ReadOnly[str]
+
 
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
@@ -155,6 +186,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         headers: dict,
     ) -> dict:
         remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
+        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
         request_params: Final = (
             {
                 **response_api_optional_request_params,
@@ -168,7 +200,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return super().transform_responses_api_request(
             model=model,
-            input=remaining_input,
+            input=normalized_input,
             response_api_optional_request_params=request_params,
             litellm_params=litellm_params,
             headers=headers,
@@ -210,6 +242,91 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
 
+    @staticmethod
+    def _agent_message_text(item: "Mapping[str, Any]") -> str:
+        content: Final = item.get("content")
+        if not isinstance(content, list):
+            return ""
+        return "".join(
+            str(block.get("text") or block.get("encrypted_content") or "")
+            for block in content
+            if isinstance(block, dict)
+        )
+
+    @classmethod
+    def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
+        text: Final = cls._agent_message_text(item)
+        if not text:
+            return None
+        rewritten: Final[_RewrittenAssistantMessageItem] = {
+            "type": "message",
+            "role": "assistant",
+            "content": ({"type": "output_text", "text": text},),
+        }
+        return rewritten
+
+    @staticmethod
+    def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
+        encrypted_content: Final = item.get("encrypted_content")
+        if not isinstance(encrypted_content, str) or not encrypted_content:
+            return None
+        rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
+        return rewritten
+
+    @staticmethod
+    def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
+        call_id: Final = item.get("call_id")
+        if not isinstance(call_id, str) or not call_id:
+            return None
+        action: Final = item.get("action")
+        rewritten: Final[_RewrittenFunctionCallItem] = {
+            "type": "function_call",
+            "call_id": call_id,
+            "name": "local_shell",
+            "arguments": json.dumps(action) if isinstance(action, dict) else "{}",
+        }
+        return rewritten
+
+    @classmethod
+    def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
+        """Returns (normalized item or None to drop it, original type when rewritten)."""
+        if not isinstance(item, dict):
+            return item, None
+        item_type: Final = item.get("type")
+        if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
+            return cls._normalize_agent_message_item(item), item_type
+        if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
+            return cls._normalize_context_compaction_item(item), item_type
+        if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
+            return cls._normalize_local_shell_call_item(item), item_type
+        return item, None
+
+    @classmethod
+    def _normalize_codex_input_items(
+        cls,
+        input: "str | ResponseInputParam",
+    ) -> "str | ResponseInputParam":
+        """Rewrite Codex history item types Mantle rejects with 400 "Invalid
+        'input': value did not match any expected variant" into supported
+        equivalents. `agent_message` (Codex multi-agent traffic; its
+        encrypted_content slot carries the plaintext payload when the model
+        never issued encrypted args) becomes an assistant message,
+        `context_compaction` becomes the `compaction` spelling Mantle accepts,
+        and `local_shell_call` becomes the function_call its recorded
+        function_call_output already pairs with.
+        """
+        if not isinstance(input, list):
+            return input
+        normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
+        rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
+        if rewritten_types:
+            verbose_logger.warning(
+                "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
+                rewritten_types,
+            )
+        kept: Final = [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+        return kept  # pyright: ignore[reportReturnType]  # Codex passthrough items sit outside the OpenAI input union
+
     def map_openai_params(
         self,
         response_api_optional_params: ResponsesAPIOptionalRequestParams,

litellm/llms/cerebras/chat.py

@@ -68,6 +68,8 @@ class CerebrasConfig(OpenAIGPTConfig):
             "tool_choice",
             "tools",
             "user",
+            "max_retries",
+            "extra_headers",
         ]
 
         # Only add reasoning_effort for models that support it

litellm/llms/cohere/rerank/transformation.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -81,6 +82,7 @@ class CohereRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("COHERE_API_KEY") or get_secret_str("CO_API_KEY") or litellm.cohere_key

litellm/llms/custom_httpx/http_handler.py

@@ -9,7 +9,7 @@ import threading
 import time
 from collections.abc import AsyncIterable, Callable, Iterable, Mapping
 from http.cookiejar import CookieJar, DefaultCookiePolicy
-from typing import TYPE_CHECKING, Any, Final, Optional, TypeAlias, TypedDict
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, TypeAlias, TypedDict
 
 import certifi
 import httpx
@@ -933,11 +933,83 @@ class AsyncHTTPHandler:
         response.raise_for_status()
         return response
 
+    # Strong references to finalizer-scheduled client-close tasks. A bare
+    # create_task() result may be garbage-collected before it runs, leaving
+    # the underlying aiohttp session unclosed ("Unclosed client session").
+    # Mirrors LiteLLMAiohttpTransport._background_close_tasks.
+    _finalizer_close_tasks: ClassVar[set["asyncio.Task[None]"]] = set()  # mutable-ok: strong refs for pending closes
+
+    @classmethod
+    def _on_finalizer_close_done(cls, task: "asyncio.Task[None]") -> None:
+        cls._finalizer_close_tasks.discard(task)
+        if task.cancelled():
+            return
+        exc: Final = task.exception()
+        if exc is not None:
+            verbose_logger.debug("Error closing client at finalization: %s", exc)
+
+    def _aiohttp_session_bound_elsewhere(self, loop: asyncio.AbstractEventLoop) -> bool:
+        """True when the wrapped aiohttp session is bound to a loop other than
+        ``loop`` — awaiting ``aclose()`` here would touch that loop's internals."""
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return False
+        session: Final = transport.client
+        if not isinstance(session, ClientSession) or session.closed:
+            return False
+        return getattr(session, "_loop", None) is not loop
+
+    def _dispose_wrapped_aiohttp_session(self) -> None:
+        """Dispose the wrapped aiohttp session when ``aclose()`` cannot run here.
+
+        Finalization either has no running loop, or a loop the session is not
+        bound to. Delegating to the transport's lifecycle-aware disposal picks
+        the safe path per session state (async close on its own loop, threadsafe
+        handoff to a loop running elsewhere, or the synchronous connector
+        teardown that flips the flags ``ClientSession.__del__`` checks), so no
+        "Unclosed client session" / "Unclosed connector" warnings fire at
+        garbage collection.
+        """
+        from litellm.llms.custom_httpx.aiohttp_transport import (
+            LiteLLMAiohttpTransport,
+        )
+
+        transport: Final = getattr(self._client, "_transport", None)
+        if not isinstance(transport, LiteLLMAiohttpTransport):
+            return
+        # A shared session (e.g. the proxy's) is never this handler's to close.
+        if not getattr(transport, "_owns_session", False):
+            return
+        session: Final = transport.client
+        if isinstance(session, ClientSession) and not session.closed:
+            transport._close_recycled_session(session)  # pyright: ignore[reportPrivateUsage]  # deliberate reuse of the transport's lifecycle-aware disposal; an async close can never run in this context
+
     def __del__(self) -> None:
         try:
             if not _handler_may_close_client(sys.getrefcount(self._client), self._owns_client):
                 return
-            asyncio.get_running_loop().create_task(self._client.aclose())
+            try:
+                loop: Final = asyncio.get_running_loop()
+            except RuntimeError:
+                # No running loop at finalization time (worker threads after
+                # their loop closed, interpreter/worker shutdown, GC in a
+                # sync context). An async close can never run here.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            if self._aiohttp_session_bound_elsewhere(loop):
+                # GC ran on a live loop (e.g. the app's) but the session
+                # belongs to another, possibly dead, loop — awaiting aclose()
+                # here is the cross-loop path the transport refuses.
+                self._dispose_wrapped_aiohttp_session()
+                return
+            task: Final = loop.create_task(self._client.aclose())
+            cls: Final = type(self)
+            cls._finalizer_close_tasks.add(task)
+            task.add_done_callback(cls._on_finalizer_close_done)
         except Exception:
             pass
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypedDict, Type
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 
 import httpx
+from httpx._types import FileContent
 from openai.types.file_deleted import FileDeleted
 
 import litellm
@@ -24,6 +25,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
     validated_max_agentic_loops,
 )
 from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
 from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
@@ -1108,6 +1110,7 @@ class BaseLLMHTTPHandler:
             headers=headers or {},
             model=model,
             optional_params=optional_rerank_params,
+            litellm_params=litellm_params,
         )
 
         api_base = provider_config.get_complete_url(
@@ -1200,6 +1203,7 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 data=json.dumps(request_data),
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
@@ -1844,6 +1848,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def async_search(
@@ -1942,6 +1947,7 @@ class BaseLLMHTTPHandler:
         return provider_config.transform_search_response(
             raw_response=response,
             logging_obj=logging_obj,
+            optional_params=optional_params,
         )
 
     async def _async_post_anthropic_messages_with_http_error_retry(
@@ -2262,6 +2268,10 @@ class BaseLLMHTTPHandler:
                 AgenticAnthropicStreamingIterator,
             )
 
+            held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
+                logging_obj=logging_obj,
+                tools=anthropic_messages_optional_request_params.get("tools"),
+            )
             initial_response = AgenticAnthropicStreamingIterator(
                 completion_stream=completion_stream,
                 http_handler=self,
@@ -2272,6 +2282,8 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
+                hold_back=bool(held_back_tool_names),
+                server_fulfilled_tool_names=held_back_tool_names,
             )
             return AnthropicMessagesStreamingResponse(
                 completion_stream=initial_response,
@@ -5119,6 +5131,20 @@ class BaseLLMHTTPHandler:
                 return True
         return False
 
+    @staticmethod
+    def _server_fulfilled_tools_in_request(logging_obj: LiteLLMLoggingObj, tools: object) -> frozenset[str]:
+        """The request's tools that a registered callback fulfills server-side (e.g. ``headroom_retrieve``)."""
+        if not isinstance(tools, list) or not tools:
+            return frozenset()
+        from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name
+
+        return frozenset(
+            name
+            for cb in _custom_logger_callbacks(logging_obj)
+            for name in getattr(cb, "server_fulfilled_tool_names", frozenset())
+            if has_tool_with_name(tools, name)
+        )
+
     @staticmethod
     def _check_agentic_loop_safety(
         tool_calls: object,
@@ -7050,9 +7076,7 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            # Use JSON when no files, otherwise use form data with files
             if files and len(files) > 0:
-                # Use multipart/form-data when files are present
                 response = sync_httpx_client.post(
                     url=api_base,
                     headers=headers,
@@ -7060,9 +7084,14 @@ class BaseLLMHTTPHandler:
                     files=files,
                     timeout=timeout,
                 )
-
+            elif video_generation_provider_config.use_multipart_form_data():
+                response = sync_httpx_client.post(  # rebind-ok: one of three mutually-exclusive branches
+                    url=api_base,
+                    headers=headers,
+                    files=serialize_multipart_form_fields(data),
+                    timeout=timeout,
+                )
             else:
-                # Use JSON content type for POST requests without files
                 response = sync_httpx_client.post(
                     url=api_base,
                     headers=headers,
@@ -7154,20 +7183,26 @@ class BaseLLMHTTPHandler:
         )
 
         try:
-            # Use JSON when no files, otherwise use form data with files
-            if files is None or len(files) == 0:
+            if files and len(files) > 0:
                 response = await async_httpx_client.post(
                     url=api_base,
                     headers=headers,
-                    json=data,
+                    data=data,
+                    files=files,
+                    timeout=timeout,
+                )
+            elif video_generation_provider_config.use_multipart_form_data():
+                response = await async_httpx_client.post(  # rebind-ok: one of three mutually-exclusive branches
+                    url=api_base,
+                    headers=headers,
+                    files=serialize_multipart_form_fields(data),
                     timeout=timeout,
                 )
             else:
                 response = await async_httpx_client.post(
                     url=api_base,
                     headers=headers,
-                    data=data,
-                    files=files,
+                    json=data,
                     timeout=timeout,
                 )
 
@@ -7827,6 +7862,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -7838,6 +7874,7 @@ class BaseLLMHTTPHandler:
             return self.async_video_edit_handler(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 video_provider_config=video_provider_config,
                 custom_llm_provider=custom_llm_provider,
                 litellm_params=litellm_params,
@@ -7891,9 +7928,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -7912,11 +7950,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = sync_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = (
+                sync_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else sync_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(
@@ -7936,6 +7973,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str,
         litellm_params,
         logging_obj,
+        video_file: FileContent | None = None,
         extra_headers: dict[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
         timeout: float | None = None,
@@ -7987,9 +8025,10 @@ class BaseLLMHTTPHandler:
             prefetched_source_data = prefetch_resp.json()
 
         try:
-            url, data = video_provider_config.transform_video_edit_request(
+            url, data, files = video_provider_config.transform_video_edit_request(
                 prompt=prompt,
                 video_id=video_id,
+                video_file=video_file,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 headers=headers,
@@ -8008,11 +8047,10 @@ class BaseLLMHTTPHandler:
                 },
             )
 
-            response: Final = await async_httpx_client.post(
-                url=url,
-                headers=headers,
-                json=data,
-                timeout=timeout,
+            response: Final = await (
+                async_httpx_client.post(url=url, headers=headers, data=data, files=files, timeout=timeout)
+                if files
+                else async_httpx_client.post(url=url, headers=headers, json=data, timeout=timeout)
             )
             response.raise_for_status()
             return video_provider_config.transform_video_edit_response(

litellm/llms/dashscope/rerank/transformation.py

@@ -22,6 +22,7 @@ as supported only for gte-rerank-v2 / qwen3-vl-rerank.
 Docs - https://help.aliyun.com/zh/model-studio/text-rerank-api
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -85,6 +86,7 @@ class DashScopeRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("DASHSCOPE_API_KEY")

litellm/llms/databricks/cost_calculator.py

@@ -3,10 +3,31 @@ Helper util for handling databricks-specific cost calculation
 - e.g.: handling 'dbrx-instruct-*'
 """
 
+from types import MappingProxyType
 from typing import Final
 
+from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
 from litellm.types.utils import Usage
-from litellm.utils import get_model_info
+
+_LEGACY_ENDPOINT_NAMES: Final = MappingProxyType(
+    {
+        "dbrx-instruct": "databricks-dbrx-instruct",
+        "meta-llama-3.1-70b-instruct": "databricks-meta-llama-3-1-70b-instruct",
+        "meta-llama-3.1-405b-instruct": "databricks-meta-llama-3-1-405b-instruct",
+        "mixtral-8x7b-instruct-v0.1": "databricks-mixtral-8x7b-instruct",
+        "bge-large-en": "databricks-bge-large-en",
+        "gte-large-en": "databricks-gte-large-en",
+        "llama-2-70b-chat": "databricks-llama-2-70b-chat",
+    }
+)
+
+
+def _registry_key(model: str) -> str:
+    name: Final = model.removeprefix("databricks/")
+    return next(
+        (key for prefix, key in _LEGACY_ENDPOINT_NAMES.items() if name.startswith(prefix)),
+        name,
+    )
 
 
 def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
@@ -20,36 +41,8 @@ def cost_per_token(model: str, usage: Usage) -> tuple[float, float]:
     Returns:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd
     """
-    base_model = model
-    if model.startswith("databricks/dbrx-instruct") or model.startswith("dbrx-instruct"):
-        base_model = "databricks-dbrx-instruct"
-    elif model.startswith("databricks/meta-llama-3.1-70b-instruct") or model.startswith("meta-llama-3.1-70b-instruct"):
-        base_model = "databricks-meta-llama-3-1-70b-instruct"
-    elif model.startswith("databricks/meta-llama-3.1-405b-instruct") or model.startswith(
-        "meta-llama-3.1-405b-instruct"
-    ):
-        base_model = "databricks-meta-llama-3-1-405b-instruct"
-    elif (
-        model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
-        or model.startswith("mixtral-8x7b-instruct-v0.1")
-        or model.startswith("databricks/mixtral-8x7b-instruct-v0.1")
-        or model.startswith("mixtral-8x7b-instruct-v0.1")
-    ):
-        base_model = "databricks-mixtral-8x7b-instruct"
-    elif model.startswith("databricks/bge-large-en") or model.startswith("bge-large-en"):
-        base_model = "databricks-bge-large-en"
-    elif model.startswith("databricks/gte-large-en") or model.startswith("gte-large-en"):
-        base_model = "databricks-gte-large-en"
-    elif model.startswith("databricks/llama-2-70b-chat") or model.startswith("llama-2-70b-chat"):
-        base_model = "databricks-llama-2-70b-chat"
-    ## GET MODEL INFO
-    model_info: Final = get_model_info(model=base_model, custom_llm_provider="databricks")
-
-    ## CALCULATE INPUT COST
-
-    prompt_cost: Final[float] = usage["prompt_tokens"] * model_info["input_cost_per_token"]
-
-    ## CALCULATE OUTPUT COST
-    completion_cost: Final = usage["completion_tokens"] * model_info["output_cost_per_token"]
-
-    return prompt_cost, completion_cost
+    return generic_cost_per_token(
+        model=_registry_key(model),
+        usage=usage,
+        custom_llm_provider="databricks",
+    )

litellm/llms/deepinfra/rerank/transformation.py

@@ -2,6 +2,7 @@
 Translate between Cohere's `/rerank` format and Deepinfra's `/rerank` format.
 """
 
+from collections.abc import Mapping
 from typing import Any, Final
 
 import httpx
@@ -67,6 +68,7 @@ class DeepinfraRerankConfig(BaseRerankConfig):
         model: str,
         api_key: str | None = None,
         optional_params: dict | None = None,
+        litellm_params: Mapping[str, object] | None = None,
     ) -> dict:
         if api_key is None:
             api_key = get_secret_str("DEEPINFRA_API_KEY")