Merge remote-tracking branch 'upstream/litellm_internal_staging' into feat/friendli-model-metadata-sync

# Conflicts:
#	model_prices_and_context_window.json

.circleci/config.yml

@@ -430,7 +430,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -504,7 +504,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise \
                 --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
@@ -631,7 +631,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -651,126 +651,6 @@ jobs:
             - auth_ui_unit_tests_coverage.xml
             - auth_ui_unit_tests_coverage
 
-  proxy_behavior_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy management behavior tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_behavior \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  proxy_security_tests:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Seed DB schema via prisma db push
-          command: |
-            uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Run proxy security tests
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_security_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
-  schema_migration_check:
-    docker:
-      - *python312_image
-      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
-        environment:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: litellm_test
-    working_directory: ~/project
-    environment:
-      # An empty database; the test applies every committed migration itself.
-      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - wait_for_service:
-          url: tcp://localhost:5432
-          timeout: "60"
-      - run:
-          name: Generate Prisma Client
-          command: uv run --no-sync python -m prisma generate
-      - run:
-          name: Check schema.prisma is in sync with committed migrations
-          command: |
-            mkdir -p test-results
-            uv run --no-sync python -m pytest tests/proxy_migration_tests \
-              -v --junitxml=test-results/junit.xml --durations=10
-          no_output_timeout: 15m
-      - store_test_results:
-          path: test-results
-
   litellm_router_testing: # Runs all tests with the "router" keyword
     docker:
       - *python312_image
@@ -858,7 +738,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -v -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -985,7 +865,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=20 \
                 -n 4 \
@@ -1030,7 +910,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1074,7 +954,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2 \
@@ -1120,7 +1000,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 --retries 3 --retry-delay 5"
@@ -1211,7 +1091,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1255,7 +1135,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1274,40 +1154,6 @@ jobs:
           paths:
             - search_coverage.xml
             - search_coverage
-  litellm_mapped_enterprise_tests:
-    docker:
-      - *python312_image
-    working_directory: ~/project
-    resource_class: large
-
-    steps:
-      - checkout
-      - skip_if_unrelated_changes
-      - setup_google_dns
-      - install_uv
-      - install_rust
-      - run:
-          name: Install Dependencies
-          command: |
-            uv sync --frozen --all-groups --all-extras --python 3.12
-      - setup_litellm_enterprise_pip
-      - run:
-          name: Run enterprise tests
-          command: |
-            uv run --no-sync python -m prisma generate
-            mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py")
-            echo "$TEST_FILES" | circleci tests run \
-              --verbose \
-              --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-                -v -x \
-                --junitxml=test-results/junit-enterprise.xml \
-                --durations=10 \
-                -n 4"
-          no_output_timeout: 15m
-      # Store test results
-      - store_test_results:
-          path: test-results
   batches_testing:
     docker:
       - *python312_image
@@ -1333,7 +1179,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1377,7 +1223,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 2"
@@ -1422,7 +1268,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
                 -n 4"
@@ -1501,7 +1347,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 -n 4 \
                 --junitxml=test-results/junit.xml \
                 --durations=5 \
@@ -1546,7 +1392,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5"
           no_output_timeout: 15m
@@ -1599,7 +1445,7 @@ jobs:
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
                 -vv -x -s \
-                --cov=./litellm --cov-report=xml \
+                --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
                 --junitxml=test-results/junit.xml \
                 --durations=5 -n 2 \
                 --reruns 2 --reruns-delay 1"
@@ -3105,12 +2951,6 @@ workflows:
           filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
-      - proxy_behavior_tests:
-          filters: *main_branches
-      - proxy_security_tests:
-          filters: *main_branches
-      - schema_migration_check:
-          filters: *main_branches
       - build_docker_database_image:
           filters: *main_branches
       - e2e_ui_testing:
@@ -3167,8 +3007,6 @@ workflows:
           filters: *main_branches
       - search_testing:
           filters: *main_branches
-      - litellm_mapped_enterprise_tests:
-          filters: *main_branches
       - batches_testing:
           filters: *main_branches
       - litellm_utils_testing:
@@ -3191,7 +3029,6 @@ workflows:
             - guardrails_testing
             - ocr_testing
             - search_testing
-            - litellm_mapped_enterprise_tests
             - batches_testing
             - litellm_utils_testing
             - pass_through_unit_testing

.github/CODEOWNERS

@@ -1,6 +1,10 @@
 /ui/ @yuneng-berri @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
+/ui/Dockerfile
+/ui/nginx.conf
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/ui/litellm-dashboard/tsconfig.tsbuildinfo
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
 /litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
+/.github/CODEOWNERS @yuneng-berri

.github/ci-coverage-allowlist.yml

@@ -5,24 +5,21 @@ description: >-
 
 test_paths:
   - reason: >-
-      The caching suite in tests/local_testing, which runs nowhere. Every job that globs that
-      directory either deselects it (local_testing_part1 and part2 carry `-k "... and not caching
-      and not cache"`) or keeps only another keyword (langfuse, router, assistants), and no job
-      names these files the way redis_caching_unit_tests names test_dual_cache.py. Measured
-      2026-08-20 by collecting the directory under each job's own selector: 118 tests across
-      these eight files are selected by none of them. Listed so the gap is a decision rather
-      than an accident, and so the --slices guard has a baseline to ratchet down from. Revisit
-      when tests/local_testing is ported off CircleCI, where the keyless part of this suite
-      belongs in a real job
+      What is left of the caching suite in tests/local_testing that runs nowhere. Every job that
+      globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and
+      not caching and not cache"`) or keeps only another keyword (langfuse, router, assistants),
+      and no job names these files the way redis_caching_unit_tests names test_dual_cache.py.
+      The gap was eight files and 118 tests when measured 2026-08-20; the five keyless ones now
+      run in the caching-local shard, leaving these three. Measured 2026-08-21 with no provider
+      credentials and no Redis: test_caching.py needs both (37 of 65 fail without them),
+      test_disk_cache_unit_tests.py needs OPENAI_API_KEY for 2 of its 4, and
+      test_gcs_cache_unit_tests.py needs GCS credentials for all 4. They want the keyless/live
+      split that porting tests/local_testing off CircleCI will force, not a job that is red by
+      construction
     paths:
-      - tests/local_testing/test_cache_preset_key.py
       - tests/local_testing/test_caching.py
-      - tests/local_testing/test_caching_handler.py
       - tests/local_testing/test_disk_cache_unit_tests.py
       - tests/local_testing/test_gcs_cache_unit_tests.py
-      - tests/local_testing/test_prompt_caching.py
-      - tests/local_testing/test_responses_stream_cache_keys.py
-      - tests/local_testing/test_unit_test_caching.py
   - reason: >-
       The end-to-end suite runs against a deployed proxy from its own in-cluster rig rather than
       from a pull request; it needs a live gateway and provider credentials no PR job holds
@@ -92,17 +89,6 @@ test_paths:
       - tests/integration/sandbox/test_e2b_sandbox.py
       - tests/integration/test_oci_integration.py
       - tests/integration/test_oci_proxy_integration.py
-  - reason: >-
-      A unit test for the proxy-extras package that no job invokes, while the package's other
-      tests live under tests/proxy_migration_tests. Measured 2026-08-20: 24 of its 28 tests pass
-      and the 4 in TestMigrationSQLIdempotency fail, because 13 migrations from 2026-03 onward use
-      bare CREATE TABLE, ADD COLUMN, CREATE INDEX and ADD CONSTRAINT rather than the guarded forms
-      this file requires. It also matches those keywords inside SQL comments, so two further
-      migrations are reported that are in fact fine. Wiring it up means deciding what to do about
-      the 13 first, and they cannot simply be edited: Prisma checksums an applied migration, so a
-      changed one breaks migrate deploy for existing installs
-    paths:
-      - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
   - reason: >-

.github/mutmut-coverage.rc

@@ -0,0 +1,5 @@
+# mutmut's gather_coverage() looks covered lines up by absolute path, so the
+# repo's `relative_files = true` makes every lookup miss and mutmut generates
+# zero mutants. Point COVERAGE_RCFILE here for mutation runs only.
+[run]
+relative_files = false

.github/pull_request_template.md

@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -110,8 +113,20 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Include only the tiers that have caveats; drop the empty ones
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
@@ -134,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

.github/scripts/assert_ci_coverage.py

@@ -312,8 +312,23 @@ def _matchable_names(relative_path: str) -> frozenset[str]:
     )
 
 
+def _workflow_named_tokens() -> frozenset[str]:
+    """Test tokens a GitHub Actions job names directly.
+
+    A CircleCI `-k` that deselects a file no longer means the file runs nowhere once a
+    workflow names it, so the slice check has to credit those the same way the census does.
+    """
+    return _invoked_test_tokens(
+        scalar
+        for path in _config_files()
+        if path != CIRCLECI_CONFIG
+        for scalar in _scalars(yaml.safe_load(path.read_text(encoding="utf-8")), path.name)
+    )
+
+
 def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
     slices: Final = _slices()
+    named_by_workflow: Final = _workflow_named_tokens()
     globbed: Final = tuple(
         path
         for path in _test_files()
@@ -326,6 +341,7 @@ def _deselected_everywhere(allowlist: Allowlist) -> tuple[Finding, ...]:
         )
         for path in globbed
         if not allowlist.covers_test(path)
+        and not any(_token_covers(token, path) for token in named_by_workflow)
         and not any(slice_.claims(path, _matchable_names(path)) for slice_ in slices)
     )
 

.github/scripts/e2e_egress_sentinel.py

@@ -0,0 +1,198 @@
+"""Prove an e2e replay run makes zero outbound provider calls, by counting them.
+
+`serve` pins each provider host (`--host`) to a local sink address in the hosts
+file and binds a counting listener on that address, so any connection the proxy
+or the record/replay edge opens to a real provider is redirected to the sink,
+recorded as one line in `--hits-file`, and never leaves the box. The record and
+replay edge only ever dials `127.0.0.1:<edge-port>` (a different host than the
+pinned provider names), so in a clean replay the sink sees nothing; a single hit
+means a provider call escaped the bundle. `assert-empty` turns that hit file into
+the pass/fail check.
+
+Stdlib only, so CI runs it under the system interpreter as root (binding :443 and
+editing the hosts file both need root); `--sink-address`, `--port`, and
+`--hosts-file` are injectable so it runs unprivileged against a temp hosts file on
+a high port under test.
+"""
+
+# ruff: noqa: T201  # CLI script: its stdout/stderr progress and results are the interface
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import signal
+import socket
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import Final
+
+_BLOCK_BEGIN: Final = "# BEGIN e2e-egress-sentinel"
+_BLOCK_END: Final = "# END e2e-egress-sentinel"
+
+
+@dataclass(frozen=True, slots=True)
+class ServeConfig:
+    hosts: tuple[str, ...]
+    sink_address: str
+    ports: tuple[int, ...]
+    hits_file: Path
+    hosts_file: Path
+    ready_file: Path | None
+    pid_file: Path | None
+
+
+def _pin_block(sink_address: str, hosts: tuple[str, ...]) -> str:
+    lines = "\n".join(f"{sink_address}\t{host}" for host in hosts)
+    return f"\n{_BLOCK_BEGIN}\n{lines}\n{_BLOCK_END}\n"
+
+
+def _install_pins(hosts_file: Path, sink_address: str, hosts: tuple[str, ...]) -> bytes:
+    original = hosts_file.read_bytes() if hosts_file.exists() else b""
+    hosts_file.write_bytes(original + _pin_block(sink_address, hosts).encode())
+    return original
+
+
+def _restore_pins(hosts_file: Path, original: bytes) -> None:
+    hosts_file.write_bytes(original)
+
+
+def _bind(sink_address: str, port: int) -> socket.socket:
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    listener.bind((sink_address, port))
+    listener.listen(128)
+    return listener
+
+
+@dataclass(frozen=True, slots=True)
+class _HitLog:
+    path: Path
+    _lock: threading.Lock
+
+    def record(self, *, port: int, peer: tuple[str, int]) -> None:
+        entry = json.dumps({"ts": time.time(), "port": port, "peer": list(peer)})
+        with self._lock:
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(entry + "\n")
+
+
+def _serve_socket(listener: socket.socket, port: int, hits: _HitLog, stop: threading.Event) -> None:
+    while not stop.is_set():
+        try:
+            conn, peer = listener.accept()
+        except OSError:
+            return
+        hits.record(port=port, peer=(peer[0], peer[1]))
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def serve(config: ServeConfig) -> int:
+    config.hits_file.write_text("", encoding="utf-8")
+    original_hosts = _install_pins(config.hosts_file, config.sink_address, config.hosts)
+    try:
+        listeners = tuple(_bind(config.sink_address, port) for port in config.ports)
+    except OSError as exc:
+        _restore_pins(config.hosts_file, original_hosts)
+        print(f"egress sentinel could not bind a sink: {exc}", file=sys.stderr)
+        return 1
+
+    stop = threading.Event()
+    hits = _HitLog(path=config.hits_file, _lock=threading.Lock())
+    threads = tuple(
+        threading.Thread(target=_serve_socket, args=(listener, port, hits, stop), daemon=True)
+        for listener, port in zip(listeners, config.ports)
+    )
+    for thread in threads:
+        thread.start()
+
+    def _handle(_signum: int, _frame: FrameType | None) -> None:
+        stop.set()
+        for listener in listeners:
+            try:
+                listener.close()
+            except OSError:
+                pass
+
+    signal.signal(signal.SIGTERM, _handle)
+    signal.signal(signal.SIGINT, _handle)
+
+    if config.pid_file is not None:
+        config.pid_file.write_text(str(os.getpid()), encoding="utf-8")
+    if config.ready_file is not None:
+        config.ready_file.write_text("ready", encoding="utf-8")
+    print(
+        f"egress sentinel up: pinned {', '.join(config.hosts)} to {config.sink_address} "
+        f"on port(s) {', '.join(str(p) for p in config.ports)}",
+        flush=True,
+    )
+
+    stop.wait()
+    _restore_pins(config.hosts_file, original_hosts)
+    if config.ready_file is not None and config.ready_file.exists():
+        config.ready_file.unlink()
+    if config.pid_file is not None and config.pid_file.exists():
+        config.pid_file.unlink()
+    return 0
+
+
+def assert_empty(hits_file: Path) -> int:
+    if not hits_file.exists():
+        print(f"egress sentinel recorded no provider calls ({hits_file} absent): zero egress")
+        return 0
+    hits = [line for line in hits_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    if not hits:
+        print("egress sentinel recorded no provider calls: zero egress")
+        return 0
+    print(f"egress sentinel recorded {len(hits)} provider call(s); replay was not hermetic:", file=sys.stderr)
+    for line in hits:
+        print(f"  {line}", file=sys.stderr)
+    return 1
+
+
+def _serve_from_args(args: argparse.Namespace) -> int:
+    config = ServeConfig(
+        hosts=tuple(args.host),
+        sink_address=args.sink_address,
+        ports=tuple(args.port),
+        hits_file=Path(args.hits_file),
+        hosts_file=Path(args.hosts_file),
+        ready_file=Path(args.ready_file) if args.ready_file else None,
+        pid_file=Path(args.pid_file) if args.pid_file else None,
+    )
+    return serve(config)
+
+
+def main(argv: tuple[str, ...]) -> int:
+    parser = argparse.ArgumentParser(description="count outbound provider calls during an e2e replay")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    serve_parser = sub.add_parser("serve", help="pin provider hosts and count connection attempts")
+    serve_parser.add_argument("--host", action="append", required=True, help="provider host to pin and watch")
+    serve_parser.add_argument("--sink-address", default="127.0.0.1")
+    serve_parser.add_argument("--port", action="append", type=int, default=None)
+    serve_parser.add_argument("--hits-file", required=True)
+    serve_parser.add_argument("--hosts-file", default="/etc/hosts")
+    serve_parser.add_argument("--ready-file", default=None)
+    serve_parser.add_argument("--pid-file", default=None)
+
+    assert_parser = sub.add_parser("assert-empty", help="exit non-zero if any provider call was recorded")
+    assert_parser.add_argument("--hits-file", required=True)
+
+    args = parser.parse_args(argv)
+    if args.command == "serve":
+        if args.port is None:
+            args.port = [443]
+        return _serve_from_args(args)
+    return assert_empty(Path(args.hits_file))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(tuple(sys.argv[1:])))

.github/scripts/e2e_fetch_fixture_bundle.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO="${1:-${GITHUB_REPOSITORY:?REPO required}}"
+ARTIFACT_NAME="${2:-e2e-fixtures-bundle}"
+BASE_BRANCH="${3:?base branch required}"
+DEST_DIR="${4:?destination bundle dir required}"
+
+: "${GH_TOKEN:?GH_TOKEN required to query and download artifacts}"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "${WORKDIR}"' EXIT
+
+echo "resolving newest non-expired '${ARTIFACT_NAME}' artifact on ${REPO}@${BASE_BRANCH}"
+
+SELECTED="$(
+  gh api "repos/${REPO}/actions/artifacts" -X GET -f per_page=100 --paginate \
+    --jq ".artifacts[] | select(.name == \"${ARTIFACT_NAME}\" and .expired == false and .workflow_run.head_branch == \"${BASE_BRANCH}\") | {id, digest, created_at, run_id: .workflow_run.id, run_number: .workflow_run.run_number}" \
+    | jq -s 'sort_by(.created_at) | reverse | .[0] // empty'
+)"
+
+if [[ -z "${SELECTED}" ]]; then
+  echo "no usable '${ARTIFACT_NAME}' artifact on ${BASE_BRANCH}: the last record run produced none (a red Saturday), so there is nothing fresh to replay; failing loudly instead of replaying a stale bundle" >&2
+  exit 1
+fi
+
+RUN_ID="$(echo "${SELECTED}" | jq -r '.run_id')"
+RUN_NUMBER="$(echo "${SELECTED}" | jq -r '.run_number')"
+ARTIFACT_ID="$(echo "${SELECTED}" | jq -r '.id')"
+GH_DIGEST="$(echo "${SELECTED}" | jq -r '.digest // "unknown"')"
+CREATED_AT="$(echo "${SELECTED}" | jq -r '.created_at')"
+
+echo "pinned bundle: run #${RUN_NUMBER} (run_id=${RUN_ID}, artifact_id=${ARTIFACT_ID}), recorded ${CREATED_AT}, github digest ${GH_DIGEST}"
+
+gh run download "${RUN_ID}" --repo "${REPO}" -n "${ARTIFACT_NAME}" -D "${WORKDIR}"
+
+TARBALL="$(find "${WORKDIR}" -name '*.tar.gz' -type f | head -n 1)"
+if [[ -z "${TARBALL}" ]]; then
+  echo "downloaded artifact contained no tarball" >&2
+  exit 1
+fi
+SIDECAR="${TARBALL}.sha256"
+if [[ ! -f "${SIDECAR}" ]]; then
+  echo "downloaded artifact has no ${SIDECAR}: cannot verify the bundle digest" >&2
+  exit 1
+fi
+
+echo "verifying bundle against its recorded sha256 digest"
+( cd "$(dirname "${TARBALL}")" && sha256sum -c "$(basename "${SIDECAR}")" )
+
+mkdir -p "${DEST_DIR}"
+tar xzf "${TARBALL}" -C "${DEST_DIR}"
+
+echo "extracted bundle into ${DEST_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  recorded_at', m['recorded_at'], 'harness', m['harness_version'], 'format_version', m['format_version'])" "${DEST_DIR}/manifest.json"

.github/scripts/e2e_pack_fixture_bundle.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 2 ]]; then
+  echo "usage: $0 <bundle-dir> <out-tarball>" >&2
+  exit 2
+fi
+
+BUNDLE_DIR="$1"
+OUT_TARBALL="$2"
+
+MANIFEST="${BUNDLE_DIR}/manifest.json"
+if [[ ! -f "${MANIFEST}" ]]; then
+  echo "no ${MANIFEST}: refusing to publish a bundle with no manifest (record produced nothing)" >&2
+  exit 1
+fi
+
+echo "packing fixture bundle from ${BUNDLE_DIR}"
+python3 -c "import json,sys; m=json.load(open(sys.argv[1])); print('  format_version', m['format_version'], 'recorded_at', m['recorded_at'], 'harness', m['harness_version'])" "${MANIFEST}"
+
+TEST_DIRS=$(find "${BUNDLE_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l | tr -d ' ')
+if [[ "${TEST_DIRS}" -eq 0 ]]; then
+  echo "bundle at ${BUNDLE_DIR} has a manifest but no recorded interactions; refusing to publish an empty bundle" >&2
+  exit 1
+fi
+echo "  ${TEST_DIRS} recorded test director(ies)"
+
+mkdir -p "$(dirname "${OUT_TARBALL}")"
+tar czf "${OUT_TARBALL}" -C "${BUNDLE_DIR}" .
+
+OUT_DIR="$(cd "$(dirname "${OUT_TARBALL}")" && pwd)"
+OUT_BASE="$(basename "${OUT_TARBALL}")"
+( cd "${OUT_DIR}" && sha256sum "${OUT_BASE}" > "${OUT_BASE}.sha256" )
+
+echo "wrote ${OUT_TARBALL} ($(du -h "${OUT_TARBALL}" | cut -f1)) and ${OUT_BASE}.sha256"
+cat "${OUT_DIR}/${OUT_BASE}.sha256"

.github/workflows/_test-unit-base.yml

@@ -149,7 +149,7 @@ jobs:
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           else
@@ -161,7 +161,7 @@ jobs:
               --reruns-delay 1 \
               --dist="${DIST}" \
               --durations=20 \
-              --cov=./litellm \
+              --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
               --cov-config=pyproject.toml
           fi

.github/workflows/check-ui-api-types.yml

@@ -83,6 +83,24 @@ jobs:
         if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
+      - name: Regenerate the lazy OpenAPI snapshot
+        if: steps.changes.outputs.relevant == 'true'
+        run: uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
+
+      - name: Fail if the lazy OpenAPI snapshot is stale
+        if: steps.changes.outputs.relevant == 'true'
+        run: |
+          if ! git diff --exit-code -- litellm/proxy/_lazy_openapi_snapshot.json; then
+            echo "::error file=litellm/proxy/_lazy_openapi_snapshot.json::The lazy OpenAPI snapshot is out of sync with the lazily loaded routes."
+            echo ""
+            echo "A lazily loaded route or model changed without regenerating the snapshot that /openapi.json serves for unloaded features."
+            echo "To fix, run from the repo root:"
+            echo "  uv run python -m litellm.proxy._lazy_openapi_snapshot"
+            echo "then run npm run gen:api from ui/litellm-dashboard and commit both files."
+            exit 1
+          fi
+          echo "_lazy_openapi_snapshot.json is in sync with the lazily loaded routes."
+
       - name: Set up Node.js
         if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0

.github/workflows/e2e_record_replay.yml

@@ -0,0 +1,237 @@
+name: "E2E Record and Replay"
+
+on:
+  schedule:
+    - cron: "0 8 * * 6"
+    - cron: "0 8 * * 1-5"
+  workflow_dispatch:
+    inputs:
+      mode:
+        description: "record (hits real providers and publishes a fresh bundle) or replay (bundle only, zero provider egress)"
+        type: choice
+        options:
+          - record
+          - replay
+        default: record
+
+permissions:
+  contents: read
+
+jobs:
+  record:
+    name: "Record the e2e suite against real providers"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 6' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'record'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Record the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: record
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Pack the fixture bundle
+        run: |
+          .github/scripts/e2e_pack_fixture_bundle.sh tests/e2e/.fixtures "${RUNNER_TEMP}/bundle/e2e-fixtures.tar.gz"
+
+      - name: Publish the fixture bundle
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        with:
+          name: e2e-fixtures-bundle
+          path: |
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz
+            ${{ runner.temp }}/bundle/e2e-fixtures.tar.gz.sha256
+          if-no-files-found: error
+          retention-days: 30
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log
+
+  replay:
+    name: "Replay the e2e suite from the pinned bundle with zero egress"
+    if: >-
+      (github.event_name != 'schedule' || github.repository == 'BerriAI/litellm') &&
+      (github.event.schedule == '0 8 * * 1-5' ||
+       (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'replay'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: read
+      actions: read
+    services:
+      postgres:
+        image: postgres:16.6
+        env:
+          POSTGRES_USER: llmproxy
+          POSTGRES_PASSWORD: dbpassword9090
+          POSTGRES_DB: litellm
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U llmproxy"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      DATABASE_URL: postgresql://llmproxy:dbpassword9090@localhost:5432/litellm
+      LITELLM_MASTER_KEY: sk-e2e-record-replay
+      LITELLM_LOCAL_MODEL_COST_MAP: "True"
+      GH_TOKEN: ${{ github.token }}
+      OPENAI_API_KEY: sk-replay-must-never-reach-a-provider
+      ANTHROPIC_API_KEY: sk-ant-replay-must-never-reach-a-provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
+
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Fetch the pinned fixture bundle by digest
+        env:
+          BASE_BRANCH: ${{ github.ref_name }}
+        run: |
+          .github/scripts/e2e_fetch_fixture_bundle.sh \
+            "${GITHUB_REPOSITORY}" \
+            e2e-fixtures-bundle \
+            "${BASE_BRANCH}" \
+            tests/e2e/.fixtures
+
+      - name: Start the proxy
+        run: |
+          nohup uv run --no-sync litellm --config tests/e2e/gateway/record_replay_ci_config.yml --port 4000 > proxy.log 2>&1 &
+          for _ in $(seq 1 90); do
+            if curl -fs http://localhost:4000/health/liveliness > /dev/null; then
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "proxy never became live"
+          tail -n 100 proxy.log
+          exit 1
+
+      - name: Start the egress sentinel
+        run: |
+          # shellcheck disable=SC2024  # the log redirect is deliberately the runner user's, so a later non-sudo cat can read it
+          sudo python3 .github/scripts/e2e_egress_sentinel.py serve \
+            --host api.openai.com \
+            --host api.anthropic.com \
+            --hits-file "${RUNNER_TEMP}/egress-hits.jsonl" \
+            --ready-file "${RUNNER_TEMP}/egress-ready" \
+            --pid-file "${RUNNER_TEMP}/egress.pid" \
+            > "${RUNNER_TEMP}/egress-sentinel.log" 2>&1 &
+          for _ in $(seq 1 30); do
+            if [[ -f "${RUNNER_TEMP}/egress-ready" ]]; then
+              cat "${RUNNER_TEMP}/egress-sentinel.log"
+              exit 0
+            fi
+            sleep 1
+          done
+          echo "egress sentinel never became ready"
+          cat "${RUNNER_TEMP}/egress-sentinel.log"
+          exit 1
+
+      - name: Replay the replayable e2e lane
+        env:
+          E2E_FIXTURE_MODE: replay
+        run: |
+          uv run --no-sync pytest tests/e2e -m replayable --reruns 0 -v --tb=short -rA
+
+      - name: Stop the egress sentinel and assert zero provider egress
+        if: always()
+        run: |
+          if [[ -f "${RUNNER_TEMP}/egress.pid" ]]; then
+            sudo kill -TERM "$(cat "${RUNNER_TEMP}/egress.pid")" 2>/dev/null || true
+            sleep 2
+          fi
+          python3 .github/scripts/e2e_egress_sentinel.py assert-empty --hits-file "${RUNNER_TEMP}/egress-hits.jsonl"
+
+      - name: Show proxy log on failure
+        if: failure()
+        run: tail -n 300 proxy.log

.github/workflows/image-scan.yml

@@ -17,10 +17,14 @@ on:
       - backend/Dockerfile
       - backend/main.py
       - docker/component_entrypoint.sh
+      - docker/entrypoint.sh
+      - litellm/proxy/prisma_migration.py
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
       - ui/litellm-dashboard/package-lock.json
+      - ui/Dockerfile
+      - ui/nginx.conf
       - .github/workflows/image-scan.yml
   schedule:
     - cron: "41 6 * * *"
@@ -183,6 +187,35 @@ jobs:
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
 
+  ui-image:
+    name: ui-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build UI image
+        run: docker build -f ui/Dockerfile -t litellm-ui-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the UI serves offline as an arbitrary uid with a read-only root fs
+        env:
+          LITELLM_IMAGE: litellm-ui-scan:${{ github.sha }}
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_ui_image_serves_offline.py -v
+
   backend-image:
     name: backend-image
     runs-on: ubuntu-latest

.github/workflows/mutation-test.yml

@@ -87,11 +87,20 @@ jobs:
         run: |
           uv pip uninstall pytest-retry || true
 
+      # Ends before the job's own deadline so a run that outlasts the budget is
+      # still followed by the report and upload steps. mutmut saves after every
+      # mutant result, to mutants/<source path>.meta, so an interrupted run
+      # still scores the mutants it finished and export-cicd-stats can read
+      # them; a cancelled job skips those steps and publishes nothing at all.
       - name: Run mutmut
+        timeout-minutes: 300
         env:
           # Make the mutants/ sandbox win over site-packages on sys.path so the
           # trampolined files are imported instead of the installed copy.
           PYTHONPATH: ${{ github.workspace }}/mutants
+          # Without this mutmut finds no covered lines and generates 0 mutants.
+          # See the file itself for why.
+          COVERAGE_RCFILE: ${{ github.workspace }}/.github/mutmut-coverage.rc
         run: |
           set -o pipefail
           mkdir -p mutants
@@ -130,6 +139,7 @@ jobs:
             mutmut-run.log
             mutants/mutmut-stats.json
             mutants/mutmut-cicd-stats.json
+            mutants/**/*.meta
             mutants/litellm/proxy/management_endpoints/**/*.py
           if-no-files-found: warn
           retention-days: 14

.github/workflows/sync-together-ai-models.yml

@@ -0,0 +1,68 @@
+name: Sync Together AI model registry
+
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  sync_together_ai_models:
+    if: github.repository == 'BerriAI/litellm'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          ref: litellm_internal_staging
+          persist-credentials: false
+      - name: Set up uv
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+      - name: Look for an already-open sync PR
+        id: existing
+        run: |
+          open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
+            --jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
+          echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
+          if [ -n "$open_pr" ]; then
+            echo "An open sync PR already exists on branch $open_pr; skipping this run."
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}
+      - name: Run the sync
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
+        env:
+          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+      - name: Regenerate the JSON schema
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          uv run --frozen python ci_cd/generate_model_prices_schema.py
+      - name: Create a pull request when the registry changed
+        if: steps.existing.outputs.open_pr == ''
+        run: |
+          if git diff --quiet; then
+            echo "Registry already in sync; no PR needed."
+            exit 0
+          fi
+          branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git checkout -b "$branch"
+          git add model_prices_and_context_window.json \
+            litellm/model_prices_and_context_window_backup.json \
+            model_prices_and_context_window.schema.json
+          git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
+          gh auth setup-git
+          git push origin "$branch"
+          gh pr create --title "feat(models): sync together_ai model registry" \
+            --body-file "$RUNNER_TEMP/pr_body.md" \
+            --head "$branch" \
+            --base litellm_internal_staging
+        env:
+          GH_TOKEN: ${{ secrets.GH_TOKEN }}

.github/workflows/test-code-quality.yml

@@ -131,6 +131,9 @@ jobs:
       - name: check_e2e_no_raw_requests
         run: uv run --no-sync python ./tests/code_coverage_tests/check_e2e_no_raw_requests.py
 
+      - name: check_migrations_no_data_rewrites
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
+
       - name: memory_test
         run: uv run --no-sync python ./tests/code_coverage_tests/memory_test.py
 

.github/workflows/test-linting.yml

@@ -67,6 +67,17 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-lint-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-lint-
+
       - name: Clean Python cache
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-mcp.yml

@@ -60,4 +60,4 @@ jobs:
       - name: Run MCP tests
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5
+          uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5

.github/workflows/test-postgres.yml

@@ -0,0 +1,145 @@
+name: "Postgres Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  postgres:
+    name: ${{ matrix.shard }}
+    runs-on: ubuntu-latest
+    timeout-minutes: ${{ matrix.job-timeout-minutes }}
+    permissions:
+      contents: read
+
+    services:
+      postgres:
+        image: postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: proxy-behavior
+            test-path: "tests/proxy_behavior"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 25
+            job-timeout-minutes: 50
+
+          - shard: proxy-security
+            test-path: "tests/proxy_security_tests"
+            seed: db-push
+            workers: 0
+            timeout-minutes: 15
+            job-timeout-minutes: 40
+
+          - shard: schema-migration
+            test-path: "tests/proxy_migration_tests"
+            seed: none
+            workers: 0
+            timeout-minutes: 20
+            job-timeout-minutes: 45
+
+    env:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
+        with:
+          persist-credentials: false
+
+      - name: Detect relevant changes
+        id: changes
+        timeout-minutes: 2
+        uses: ./.github/actions/detect-changes
+
+      - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/setup-uv-with-retries
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-postgres-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-postgres-
+
+      - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 12
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --all-groups --all-extras
+
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
+      - name: Generate Prisma client
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Seed database schema
+        if: steps.changes.outputs.decision != 'skip' && matrix.seed != 'none'
+        timeout-minutes: 10
+        run: |
+          uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+
+      - name: Run tests
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ matrix.timeout-minutes }}
+        env:
+          TEST_PATH: ${{ matrix.test-path }}
+          WORKERS: ${{ matrix.workers }}
+        run: |
+          if [ "${WORKERS}" = "0" ]; then
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10
+          else
+            uv run --no-sync pytest ${TEST_PATH:?} -vv --tb=short --durations=10 -n "${WORKERS}"
+          fi

.github/workflows/test-unit.yml

@@ -164,11 +164,12 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers
               tests/test_litellm/proxy/utils
-            workers: 2
+            workers: 4
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
@@ -195,11 +196,40 @@ jobs:
               tests/test_litellm/proxy/types_utils
               tests/test_litellm/proxy/logging_endpoints
               tests/test_litellm/proxy/test_*.py
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: caching-local
+            artifact-name: caching-local
+            test-path: >-
+              tests/local_testing/test_cache_preset_key.py
+              tests/local_testing/test_caching_handler.py
+              tests/local_testing/test_prompt_caching.py
+              tests/local_testing/test_responses_stream_cache_keys.py
+              tests/local_testing/test_unit_test_caching.py
             workers: 2
             reruns: 2
             timeout-minutes: 20
             job-timeout-minutes: 60
 
+          - shard: proxy-extras
+            artifact-name: proxy-extras
+            test-path: "tests/litellm-proxy-extras"
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
+          - shard: enterprise-package
+            artifact-name: enterprise-package
+            test-path: "tests/enterprise"
+            workers: 4
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 60
+
           - shard: responses-caching-types
             artifact-name: responses-caching-types
             test-path: >-

.gitignore

@@ -3,6 +3,8 @@
 tests/e2e/.fixtures/
 .venv-typecheck
 .venv_policy_test
+.venv-mutmut
+mutants/
 .env
 .claude
 CLAUDE.local.md

CLAUDE.md

@@ -37,13 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 
@@ -65,6 +66,8 @@ Commit and push your work when you're done without asking
 
 When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
 
+Always pull before starting any work. The checkout or worktree may be sitting on a stale branch
+
 If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
 
 Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch
@@ -79,6 +82,8 @@ Do not put names of customers or customer company names in code, PR descriptions
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Prisma migrations apply synchronously at proxy boot, before it serves traffic, so a migration must only change schema, never rewrite rows. No `UPDATE`, `DELETE` or `MERGE`, and no `INSERT ... SELECT`: on a spend-log-sized table any of those is minutes of downtime plus a doubled heap that plain autovacuum won't give back. `tests/code_coverage_tests/check_migrations_no_data_rewrites.py` enforces this. When a rewrite is genuinely bounded and has to ship inside the migration, mark the statement `-- data-migration-ok: <what bounds it>`
+
 Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
 
 - Composition over inheritance

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

basedpyright-code-budget.json

@@ -1,18 +1,18 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 18483
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2564
   },
   "reportAssignmentType": {
-    "limit": 320
+    "limit": 319
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 480
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 113
   },
   "reportConstantRedefinition": {
     "limit": 40
@@ -24,13 +24,13 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 5960
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 154
+    "limit": 105
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -45,7 +45,7 @@
     "limit": 35
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 34
   },
   "reportInvalidTypeVarUse": {
     "limit": 2
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5659
   },
   "reportMissingTypeArgument": {
-    "limit": 15555
+    "limit": 15484
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 1058
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1822
+    "limit": 1808
   },
   "reportRedeclaration": {
     "limit": 8
@@ -99,31 +99,31 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44655
+    "limit": 44526
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 38782
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19829
   },
   "reportUnknownVariableType": {
-    "limit": 30569
+    "limit": 30349
   },
   "reportUnnecessaryCast": {
     "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 697
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 831
   },
   "reportUntypedBaseClass": {
     "limit": 0
@@ -135,12 +135,12 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 139
+    "limit": 138
   },
   "reportUnusedImport": {
-    "limit": 545
+    "limit": 544
   },
   "reportUnusedVariable": {
-    "limit": 146
+    "limit": 145
   }
 }

ci_cd/generate_model_prices_schema.py

@@ -73,6 +73,11 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
         "description": "Output modalities the model can produce.",
         "items": {"type": "string", "enum": ["text", "image", "audio", "video", "code"]},
     },
+    "reasoning_effort_levels": {
+        "type": "array",
+        "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.",
+        "items": {"type": "string", "enum": ["none", "minimal", "low", "medium", "high", "xhigh", "max"]},
+    },
     "supported_regions": {
         "type": "array",
         "description": "Cloud regions the model is available in ('global' or region ids).",
@@ -157,6 +162,9 @@ COST_DESCRIPTIONS: dict[str, str] = {
     "input_cost_per_token": "USD per prompt token.",
     "output_cost_per_token": "USD per generated token.",
     "output_cost_per_reasoning_token": "USD per reasoning/thinking token, when billed separately.",
+    "google_maps_grounding_cost_per_query": (
+        "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit."
+    ),
     "cache_creation_input_token_cost": "USD per token written to the provider's prompt cache.",
     "cache_read_input_token_cost": "USD per prompt token served from the provider's prompt cache.",
     "input_cost_per_token_batches": "USD per prompt token via the provider's batch API.",

db_scripts/partition_spend_logs.sql

@@ -10,6 +10,11 @@
 -- partitioned, so existing installs are unaffected until you run this.
 --
 -- IMPORTANT
+--   * After partitioning, `prisma db push` (including the proxy's
+--     --use_prisma_db_push startup mode) is NOT supported: it tries to rewrite
+--     the primary key back to ("request_id"), which Postgres rejects on a
+--     partitioned table. The proxy detects this and exits with guidance.
+--     Use the default startup path (`prisma migrate deploy`) instead.
 --   * Test on a staging copy first and take a backup.
 --   * Postgres cannot convert a populated table to partitioned in place, so this
 --     renames the old table aside and creates a fresh partitioned table.

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -14,6 +14,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -351,7 +353,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
@@ -966,6 +968,16 @@ class CheckBatchCost:
                     )
 
             elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                from litellm.proxy.openai_files_endpoints.common_utils import (
+                    _completed_batch_safe_to_retire,
+                )
+
+                if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
+                    verbose_proxy_logger.info(
+                        f"CheckBatchCost: batch {batch_id} is completed but its output file id "
+                        f"has not appeared yet; leaving job {job.id} for the next poll cycle"
+                    )
+                    continue
                 await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,6 +1,8 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by the get-responses call.
+Cost tracking is handled by the get-responses call, which prices normally only because the
+poll stamps itself with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN; user-facing reads of the
+same route are non-inference and free.
 """
 
 from datetime import datetime, timedelta, timezone
@@ -9,12 +11,14 @@ from typing import TYPE_CHECKING, Dict, Optional, cast
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
     MANAGED_OBJECT_STALENESS_CUTOFF_DAYS,
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -113,7 +117,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by the get-responses call
+        - Cost is tracked by the get-responses call, billed because the poll is stamped
+          with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
         - Mark responses in a terminal state as complete in the database
         """
         try:
@@ -153,6 +158,7 @@ class CheckResponsesCost:
                 # Prepare metadata with model information for cost tracking
                 litellm_metadata = {
                     "user_api_key_user_id": job.created_by or "default-user-id",
+                    INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,
                 }
                 
                 # Add model information if available

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -45,6 +45,8 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
+    FILE_LIST_CONTINUATION_CHUNK_SIZE,
+    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
     ensure_batch_response_managed_file_ids,
@@ -54,6 +56,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
+    validate_file_list_limit,
+    validate_file_list_purpose,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
     request_tags_from_metadata,
@@ -63,9 +67,9 @@ from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccess
     AsyncCursorPage,
     ChatCompletionFileObject,
     CreateFileRequest,
+    FileListPage,
     FileObject,
     OpenAIFileObject,
-    OpenAIFilesPurpose,
     ResponsesAPIResponse,
 )
 from litellm.types.utils import (
@@ -144,7 +148,14 @@ class _ManagedFileRow(Protocol):
 class _ManagedFileTableActions(Protocol):
     async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
 
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
+    async def find_many(
+        self,
+        where: Mapping[str, object],
+        take: int = ...,
+        order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
+        cursor: Mapping[str, str] = ...,
+        skip: int = ...,
+    ) -> Sequence[_ManagedFileRow]: ...
 
     async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
 
@@ -1365,12 +1376,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
     async def afile_list(
         self,
-        purpose: Optional[OpenAIFilesPurpose],
+        purpose: Optional[str],
         litellm_parent_otel_span: Optional[Span],
+        user_api_key_dict: UserAPIKeyAuth,
+        limit: Optional[int] = None,
+        after: Optional[str] = None,
         **data: Dict,
-    ) -> List[OpenAIFileObject]:
-        """Handled in files_endpoints.py"""
-        return []
+    ) -> FileListPage:
+        """List the managed files the caller owns, newest first.
+
+        Pagination is keyset based on ``unified_file_id`` so a key that owns
+        every file on the proxy still reads one bounded page at a time.
+        ``purpose`` is applied after parsing, because the managed file table
+        keeps it inside the ``file_object`` blob instead of a column, and rows
+        whose blob will not parse drop out there too, so a chunk of rows can
+        yield fewer matches than the page holds. Successive chunks are read
+        until the page is full or the caller's rows run out, which keeps
+        ``data`` non-empty while matches remain and its last id usable as the
+        next cursor. A first chunk that fills the page costs one query; once a
+        scan has to continue past it, the chunk widens to
+        ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
+        that many rows instead of one per page. That bound is per query, not
+        per request: the work is still linear in the rows the caller owns, and
+        a filter matching nothing reads every one of them, with no index
+        covering either the owner filter or the sort.
+        """
+        validate_file_list_limit(limit)
+        validate_file_list_purpose(purpose)
+
+        owner_filter: Final = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return FileListPage(**build_list_page([]))
+
+        if after:
+            cursor_row = await _managed_file_table(self.prisma_client).find_first(
+                where={**owner_filter, "unified_file_id": after}
+            )
+            if cursor_row is None:
+                raise ProxyException(
+                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    type="invalid_request_error",
+                    param="after",
+                    code=400,
+                    openai_code="invalid_value",
+                )
+
+        page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
+        matches: Final[List[OpenAIFileObject]] = []
+        cursor_id = after
+        chunk_size = page_size + 1
+
+        while len(matches) <= page_size:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_file_table(self.prisma_client).find_many(
+                where=owner_filter,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
+                **cursor_args,
+            )
+            matches.extend(
+                parsed_file_object.model_copy(update={"id": row.unified_file_id})
+                for row in chunk
+                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                and (purpose is None or parsed_file_object.purpose == purpose)
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_file_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+
+        return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
 
     def _is_batch_polling_enabled(self) -> bool:
         """

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -12,7 +12,7 @@ Endpoints for /project operations
 
 import json
 from collections.abc import Sequence
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request
 
@@ -35,6 +35,8 @@ if TYPE_CHECKING:
         LiteLLM_VerificationTokenActions,
     )
 
+    from litellm import Router
+
 router = APIRouter()
 
 
@@ -205,6 +207,114 @@ def _check_team_project_limits(
         )
 
 
+def _project_models_missing_positive_quota(
+    models: list[str] | None,
+    rpm_limits: Mapping[str, object] | None,
+    tpm_limits: Mapping[str, object] | None,
+) -> list[str]:
+    """Return the models that lack a positive `rpm` AND `tpm` quota.
+
+    A valid quota is a positive integer; null, zero, and negative are rejected
+    because downstream rate limiters treat a non-positive limit as immediately
+    exhausted (every request blocked).
+    """
+
+    def _is_positive(value: object) -> bool:
+        return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+    rpm = rpm_limits or {}
+    tpm = tpm_limits or {}
+    return [model for model in (models or []) if not _is_positive(rpm.get(model)) or not _is_positive(tpm.get(model))]
+
+
+def _router_access_group_names(llm_router: "Router | None") -> frozenset[str]:
+    return frozenset(llm_router.get_model_access_groups()) if llm_router is not None else frozenset()
+
+
+def _project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> tuple[str, ...]:
+    """Entries project auth expands to many concrete models (`all-proxy-models`, `*` patterns,
+    access groups). The rate limiter looks quotas up by the exact requested model name, so a
+    quota keyed on one of these entries is never applied."""
+    return tuple(
+        model
+        for model in (models or ())
+        if model == SpecialModelNames.all_proxy_models.value or "*" in model or model in access_group_names
+    )
+
+
+def _raise_on_project_models_expanding_at_request_time(
+    models: Sequence[str] | None, access_group_names: frozenset[str]
+) -> None:
+    expanding: Final = _project_models_expanding_at_request_time(models, access_group_names)
+    if not expanding:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {list(expanding)} expand to multiple models at request time, so a per-model rpm/tpm quota cannot be enforced for them while 'enforce_project_model_quota' is enabled. List concrete model names instead."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota(
+    data: NewProjectRequest | UpdateProjectRequest, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota for every model on project CREATE.
+
+    `model_rpm_limit`/`model_tpm_limit` are relocated into `metadata` by the request
+    model's `set_model_info` validator, so they are read from there.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    _raise_on_project_models_expanding_at_request_time(data.models, access_group_names)
+    metadata = data.metadata or {}
+    missing = _project_models_missing_positive_quota(
+        data.models, metadata.get("model_rpm_limit"), metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} added to project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
+def _raise_on_missing_project_model_quota_on_update(
+    data: UpdateProjectRequest, existing_project: object, access_group_names: frozenset[str] = frozenset()
+) -> None:
+    """Require a positive `rpm`/`tpm` quota over the RESULTING state on project UPDATE.
+
+    `/project/update` replaces `models` and `metadata` when they are provided, so the
+    check runs on what the project WILL look like: a partial update that doesn't touch
+    models/quota keeps the existing values, while one that adds a model or clears a
+    model's quota must leave every resulting model with a positive limit.
+
+    Only invoked when `general_settings.enforce_project_model_quota` is enabled
+    (default off), so it is opt-in and does not change behavior for existing users.
+    """
+    resulting_models = data.models if data.models is not None else (getattr(existing_project, "models", None) or [])
+    resulting_metadata = (
+        data.metadata if data.metadata is not None else (getattr(existing_project, "metadata", None) or {})
+    )
+    _raise_on_project_models_expanding_at_request_time(resulting_models, access_group_names)
+    missing = _project_models_missing_positive_quota(
+        resulting_models, resulting_metadata.get("model_rpm_limit"), resulting_metadata.get("model_tpm_limit")
+    )
+    if not missing:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": f"models {missing} would be left on the project without a positive rpm/tpm quota. Set a positive model_rpm_limit and model_tpm_limit for each model."
+        },
+    )
+
+
 async def _create_budget_for_project(
     data: NewProjectRequest,
     user_id: str | None,
@@ -352,7 +462,9 @@ async def new_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
     )
@@ -399,6 +511,10 @@ async def new_project(
             data=data,
         )
 
+        # Opt-in (default off): require rpm/tpm for every model added to the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota(data, _router_access_group_names(llm_router))
+
         # Check if user has permission to create projects for this team
         # only team admins can create projects for their team
         has_permission = await _check_user_permission_for_project(
@@ -538,7 +654,9 @@ async def update_project(
     ```
     """
     from litellm.proxy.proxy_server import (
+        general_settings,
         litellm_proxy_admin_name,
+        llm_router,
         premium_user,
         prisma_client,
         user_api_key_cache,
@@ -642,6 +760,12 @@ async def update_project(
                 data=data,
             )
 
+        # Opt-in (default off): require rpm/tpm for every model the update would leave on the project.
+        if general_settings.get("enforce_project_model_quota", False):
+            _raise_on_missing_project_model_quota_on_update(
+                data, existing_project, _router_access_group_names(llm_router)
+            )
+
         # Prepare update data
         update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.58"
+version = "0.1.61"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.58"
+version = "0.1.61"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

helm/litellm/values.yaml

@@ -428,9 +428,11 @@ ui:
     maxUnavailable: ""
   podAnnotations: {}
   # Same shape as the gateway blocks of the same name. The nginx runtime
-  # writes its pid, cache, and proxy temp files under the image's root
-  # filesystem, so `securityContext.readOnlyRootFilesystem: true` here needs
-  # emptyDir volumes mounted over those paths.
+  # writes its pid, cache, and proxy temp files under /tmp, so it boots as
+  # any (arbitrary, non-root) uid; `securityContext.readOnlyRootFilesystem:
+  # true` here needs an emptyDir volume mounted over /tmp. Images before
+  # the /tmp move instead need emptyDirs over /var/cache/nginx and /run to
+  # run as a non-root uid at all.
   podLabels: {}
   podSecurityContext: {}
   securityContext: {}

litellm-proxy-extras/litellm_proxy_extras/migrations/20260824000000_add_gateway_injected_caching_savings_spend/migration.sql

@@ -0,0 +1,18 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyOrganizationSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyEndUserSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyAgentSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN IF NOT EXISTS "gateway_injected_caching_savings_spend" DOUBLE PRECISION NOT NULL DEFAULT 0.0;
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -754,6 +754,7 @@ model LiteLLM_DailyUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -789,6 +790,7 @@ model LiteLLM_DailyOrganizationSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -824,6 +826,7 @@ model LiteLLM_DailyEndUserSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -858,6 +861,7 @@ model LiteLLM_DailyAgentSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -892,6 +896,7 @@ model LiteLLM_DailyTeamSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)
@@ -929,6 +934,7 @@ model LiteLLM_DailyTagSpend {
   compression_saved_tokens    BigInt      @default(0)
   compression_savings_spend    Float       @default(0.0)
   prompt_caching_savings_spend Float       @default(0.0)
+  gateway_injected_caching_savings_spend Float @default(0.0)
   autorouter_savings_spend     Float       @default(0.0)
   spend               Float    @default(0.0)
   api_requests        BigInt      @default(0)

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -40,6 +40,65 @@ def _get_prisma_env() -> dict:
 
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
+_SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
+_SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
+    r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
+)
+_SPEND_LOGS_PK_CLAUSE_RE = re.compile(
+    r'^(?:DROP\s+CONSTRAINT\s+"[^"]*_pkey"'
+    r'|ADD\s+(?:CONSTRAINT\s+"[^"]*"\s+)?PRIMARY\s+KEY\s*\([^)]*\))$',
+    re.IGNORECASE,
+)
+
+PARTITIONED_SPEND_LOGS_PUSH_ERROR = (
+    "LiteLLM_SpendLogs is a partitioned table (see db_scripts/partition_spend_logs.sql), "
+    "so its primary key must include the partition key (\"startTime\"). `prisma db push` "
+    "reconciles the database against schema.prisma, which declares the unpartitioned "
+    "primary key (\"request_id\"), and Postgres rejects that rewrite with: unique "
+    "constraint on partitioned table must include all partitioning columns. Start the "
+    "proxy without --use_prisma_db_push so it uses `prisma migrate deploy`, which only "
+    "applies shipped migrations and leaves the partitioned primary key alone."
+)
+
+
+def _without_sql_comments(statement: str) -> str:
+    return "\n".join(
+        line
+        for line in statement.splitlines()
+        if line.strip() and not line.strip().startswith("--")
+    ).strip()
+
+
+def _without_spend_logs_pk_clauses(statement: str) -> Optional[str]:
+    prefix_match = _SPEND_LOGS_ALTER_RE.match(statement)
+    if not prefix_match:
+        return statement
+    kept = tuple(
+        clause.strip()
+        for clause in statement[prefix_match.end():].split(",\n")
+        if not _SPEND_LOGS_PK_CLAUSE_RE.match(clause.strip())
+    )
+    if not kept:
+        return None
+    return statement[: prefix_match.end()] + ",\n".join(kept)
+
+
+def filter_partitioned_spend_logs_diff(diff_sql: str) -> str:
+    """Drop statements from a `prisma migrate diff` script that fight the
+    SpendLogs partitioning runbook (db_scripts/partition_spend_logs.sql): the
+    primary-key rewrite on "LiteLLM_SpendLogs", which Postgres rejects on a
+    partitioned table, and drops of runbook artifacts such as
+    "LiteLLM_SpendLogs_legacy"."""
+    kept = tuple(
+        filtered
+        for statement in diff_sql.split(";")
+        for bare in (_without_sql_comments(statement),)
+        if bare and not _SPEND_LOGS_ARTIFACT_DROP_RE.match(bare)
+        for filtered in (_without_spend_logs_pk_clauses(bare),)
+        if filtered is not None
+    )
+    return "".join(f"{statement};\n\n" for statement in kept)
+
 
 def _migration_timestamp(name: str) -> int:
     """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
@@ -355,7 +414,24 @@ class ProxyExtrasDBManager:
             return
         logger.info(f"Migration diff created at {diff_sql_path}")
 
+        if ProxyExtrasDBManager.spend_logs_is_partitioned():
+            filtered_sql = filter_partitioned_spend_logs_diff(
+                diff_sql_path.read_text()
+            )
+            diff_sql_path.write_text(filtered_sql)
+            logger.info(
+                "LiteLLM_SpendLogs is partitioned; removed its primary-key "
+                "rewrite and partitioning artifacts from the drift script"
+            )
+            if not filtered_sql.strip():
+                logger.info("Drift script is empty after filtering; nothing to apply")
+                if not mark_all_applied:
+                    return
+                ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+                return
+
         # 2. Run prisma db execute to apply the migration
+        applied_ok = False
         try:
             logger.info("Running prisma db execute to apply the migration diff...")
             result = subprocess.run(
@@ -376,6 +452,7 @@ class ProxyExtrasDBManager:
             )
             logger.info(f"prisma db execute stdout: {result.stdout}")
             logger.info("✅ Migration diff applied successfully")
+            applied_ok = True
         except subprocess.CalledProcessError as e:
             logger.warning(f"Failed to apply migration diff: {e.stderr}")
         except subprocess.TimeoutExpired:
@@ -384,6 +461,16 @@ class ProxyExtrasDBManager:
         # 3. Mark all migrations as applied
         if not mark_all_applied:
             return
+        if not applied_ok:
+            logger.warning(
+                "Drift script failed to apply; NOT marking migrations as "
+                "applied so a later migration run can retry them"
+            )
+            return
+        ProxyExtrasDBManager._mark_migrations_applied(migrations_dir)
+
+    @staticmethod
+    def _mark_migrations_applied(migrations_dir: str):
         migration_names = ProxyExtrasDBManager._get_migration_names(migrations_dir)
         logger.info(f"Resolving {len(migration_names)} migrations")
         for migration_name in migration_names:
@@ -410,6 +497,55 @@ class ProxyExtrasDBManager:
                         f"Failed to resolve migration {migration_name}: {e.stderr}"
                     )
 
+    @staticmethod
+    def spend_logs_is_partitioned() -> bool:
+        """True when the connected database's LiteLLM_SpendLogs is a
+        partitioned table in Prisma's target schema (the `schema` URL param,
+        falling back to Prisma's default target, public), i.e. the operator
+        ran db_scripts/partition_spend_logs.sql. Returns False when psycopg is
+        unavailable or the database cannot be reached, preserving the
+        pre-existing behavior in those cases."""
+        database_url = os.getenv("DATABASE_URL")
+        if not database_url:
+            return False
+
+        try:
+            import psycopg
+        except ImportError:
+            return False
+
+        cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        try:
+            with psycopg.connect(
+                cleaned_url, connect_timeout=10, autocommit=True
+            ) as conn:
+                row = conn.execute(
+                    "SELECT 1 "
+                    "FROM pg_partitioned_table pt "
+                    "JOIN pg_class c ON c.oid = pt.partrelid "
+                    "JOIN pg_namespace n ON n.oid = c.relnamespace "
+                    "WHERE c.relname = 'LiteLLM_SpendLogs' "
+                    "  AND n.nspname = %s",
+                    (
+                        ProxyExtrasDBManager._prisma_schema_param(database_url)
+                        or "public",
+                    ),
+                ).fetchone()
+        except (psycopg.OperationalError, psycopg.DatabaseError):
+            return False
+        return row is not None
+
+    @staticmethod
+    def _prisma_schema_param(url: str) -> Optional[str]:
+        """The `schema` query param Prisma uses to pick its target schema,
+        or None when the URL does not set one."""
+        from urllib.parse import urlparse, parse_qsl
+
+        return next(
+            (v for k, v in parse_qsl(urlparse(url).query) if k == "schema"),
+            None,
+        )
+
     @staticmethod
     def _strip_prisma_query_params(url: str) -> str:
         """Remove Prisma-specific query params (connection_limit, pool_timeout,
@@ -528,7 +664,8 @@ class ProxyExtrasDBManager:
         migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
 
         if not use_migrate:
-            # Preserve `prisma db push` path unchanged.
+            if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
             original_dir = os.getcwd()
             os.chdir(migrations_dir)
             try:
@@ -972,6 +1109,8 @@ class ProxyExtrasDBManager:
                                 )
                                 raise
                 else:
+                    if ProxyExtrasDBManager.spend_logs_is_partitioned():
+                        raise RuntimeError(PARTITIONED_SPEND_LOGS_PUSH_ERROR)
                     # Use prisma db push with increased timeout
                     subprocess.run(
                         [_get_prisma_command(), "db", "push", "--accept-data-loss"],

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.88"
+version = "0.4.90"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.88"
+version = "0.4.90"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+log_client_error_tracebacks: bool = False
 request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
@@ -444,6 +445,7 @@ max_ui_session_budget: Optional[float] = (
     1.0  # USD budget for each dashboard login session (playground, test connection)
 )
 internal_user_budget_duration: Optional[str] = None
+budget_rollover: bool = False  # carry spend beyond max_budget into the next window instead of zeroing it
 tag_budget_config: Optional[Dict[str, "BudgetConfig"]] = None
 max_end_user_budget: Optional[float] = None
 max_end_user_budget_id: Optional[str] = None
@@ -463,6 +465,11 @@ prometheus_metrics_config: Optional[List] = None
 prometheus_exclude_metrics: Optional[List[str]] = None
 prometheus_exclude_labels: Optional[List[str]] = None
 prometheus_emit_stream_label: bool = False
+prometheus_deployment_and_latency_caller_identity: Literal[
+    "api_key_alias",
+    "user_email",
+    "both",
+] = "api_key_alias"
 # Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
 # `litellm_proxy_failed_requests_metric`. Off by default to preserve the
 # pre-unification label set so existing dashboards / recording rules keyed on
@@ -1628,6 +1635,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
@@ -1801,6 +1811,9 @@ if TYPE_CHECKING:
     from .llms.gemini.interactions.transformation import (
         GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig,
     )
+    from .llms.vertex_ai.interactions.transformation import (
+        VertexAIInteractionsConfig as VertexAIInteractionsConfig,
+    )
     from .llms.openai.chat.o_series_transformation import (
         OpenAIOSeriesConfig as OpenAIOSeriesConfig,
         OpenAIOSeriesConfig as OpenAIO1Config,

litellm/_lazy_imports_registry.py

@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -242,6 +243,7 @@ LLM_CONFIG_NAMES: Final = (
     "OpenRouterResponsesAPIConfig",
     "BedrockMantleResponsesAPIConfig",
     "GoogleAIStudioInteractionsConfig",
+    "VertexAIInteractionsConfig",
     "OpenAIOSeriesConfig",
     "AnthropicSkillsConfig",
     "BaseSkillsAPIConfig",
@@ -740,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
@@ -977,6 +983,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         ".llms.gemini.interactions.transformation",
         "GoogleAIStudioInteractionsConfig",
     ),
+    "VertexAIInteractionsConfig": (
+        ".llms.vertex_ai.interactions.transformation",
+        "VertexAIInteractionsConfig",
+    ),
     "OpenAIOSeriesConfig": (
         ".llms.openai.chat.o_series_transformation",
         "OpenAIOSeriesConfig",

litellm/_logging.py

@@ -5,7 +5,7 @@ import os
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Final
+from typing import Any, Final, TextIO
 
 import litellm
 from litellm.constants import (
@@ -234,11 +234,65 @@ class CorrelationContextFilter(logging.Filter):
 _correlation_filter: Final = CorrelationContextFilter()
 
 
-json_logs = bool(os.getenv("JSON_LOGS", False))
+_LOG_FORMAT_PREFIX: Final = "%(asctime)s - %(name)s:%(levelname)s"
+_LOG_FORMAT_SUFFIX: Final = ": %(filename)s:%(lineno)s - %(message)s"
+_PLAIN_LOG_FORMAT: Final = _LOG_FORMAT_PREFIX + _LOG_FORMAT_SUFFIX
+_COLOR_LOG_FORMAT: Final = f"\033[92m{_LOG_FORMAT_PREFIX}\033[0m{_LOG_FORMAT_SUFFIX}"
+
+
+def _stream_is_tty(stream: TextIO | None) -> bool:
+    """True when the stream is an open interactive terminal; never raises.
+
+    A stream can be None (pythonw/embedded interpreters), lack isatty entirely
+    (GUI log-redirect shims), or be closed; import must survive all three.
+    """
+    try:
+        return stream is not None and stream.isatty()
+    except (AttributeError, ValueError):
+        return False
+
+
+def _plain_log_format(stdout: TextIO | None, stderr: TextIO | None) -> str:
+    """The plain-text log format, colorized only when both streams are an interactive terminal.
+
+    Honors the NO_COLOR convention from no-color.org: color is disabled when
+    NO_COLOR is present with a non-empty value.
+    """
+    if os.environ.get("NO_COLOR"):
+        return _PLAIN_LOG_FORMAT
+    return _COLOR_LOG_FORMAT if _stream_is_tty(stdout) and _stream_is_tty(stderr) else _PLAIN_LOG_FORMAT
+
+
+class LevelRoutingStreamHandler(logging.StreamHandler):
+    """Writes records below WARNING to stdout and WARNING and above to stderr.
+
+    Collectors that derive severity from the stream report every stderr line as an error.
+    """
+
+    def emit(self, record: logging.LogRecord) -> None:
+        preferred: Final = sys.stdout if record.levelno < logging.WARNING else sys.stderr
+        if preferred is None or getattr(preferred, "closed", False):
+            self.stream = sys.stderr  # rebind-ok: fall back to the pre-fix stream rather than raising per record
+        else:
+            self.stream = preferred  # rebind-ok: StreamHandler.emit writes self.stream under the handler lock
+        super().emit(record)
+
+
+def _parse_json_logs_env(value: str | None) -> bool:
+    """Strict opt-in parse for the JSON_LOGS env var: only "true" (any case) enables JSON logs.
+
+    Matches the reader in litellm-proxy-extras/_logging.py. The previous
+    bool(os.getenv(...)) treated any non-empty value, including "false" and "0",
+    as enabled.
+    """
+    return (value or "").lower() == "true"
+
+
+json_logs: Final = _parse_json_logs_env(os.getenv("JSON_LOGS"))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
 numeric_level: Final[str] = getattr(logging, log_level.upper())
-handler: Final = logging.StreamHandler()
+handler: Final = LevelRoutingStreamHandler()
 handler.setLevel(numeric_level)
 handler.addFilter(_secret_filter)
 handler.addFilter(_correlation_filter)
@@ -447,7 +501,7 @@ if json_logs:
     _setup_json_exception_handlers(JsonFormatter())
 else:
     formatter: Final = CorrelationPlainFormatter(
-        "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
+        _plain_log_format(sys.stdout, sys.stderr),
         datefmt="%H:%M:%S",
     )
 
@@ -628,7 +682,7 @@ def _turn_on_json():
 
     - Adds a JSON formatter to all loggers
     """
-    handler: Final = logging.StreamHandler()
+    handler: Final = LevelRoutingStreamHandler()
     handler.setFormatter(JsonFormatter())
     _initialize_loggers_with_handler(handler)
     # Set up exception handlers

litellm/_redis.py

@@ -12,8 +12,9 @@ import json
 
 # s/o [@Frank Colson](https://www.linkedin.com/in/frank-colson-422b9b183/) for this redis implementation
 import os
-from collections.abc import Callable
+from collections.abc import Callable, Mapping
 from typing import Final
+from urllib.parse import urlsplit, urlunsplit
 
 import redis
 import redis.asyncio as async_redis
@@ -50,6 +51,7 @@ def _get_redis_kwargs():
     include_args: Final = {
         "url",
         "redis_connect_func",
+        "credential_provider",
         "gcp_service_account",
         "gcp_ssl_ca_certs",
         "azure_redis_ad_token",
@@ -155,7 +157,8 @@ def _get_redis_cluster_kwargs(client=None):
 def _get_redis_env_kwarg_mapping():
     PREFIX: Final = "REDIS_"
 
-    return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs()}
+    exclude_from_environment: Final = frozenset({"credential_provider"})
+    return {f"{PREFIX}{x.upper()}": x for x in _get_redis_kwargs() if x not in exclude_from_environment}
 
 
 def _redis_kwargs_from_environment():
@@ -353,6 +356,12 @@ def get_redis_url_from_environment():
     return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
 
 
+def _url_without_userinfo(url: str) -> str:
+    parts: Final = urlsplit(url)
+    netloc: Final = parts.netloc.rsplit("@", 1)[-1]
+    return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+
 def _get_redis_client_logic(**env_overrides):
     """
     Common functionality across sync + async redis client implementations
@@ -410,54 +419,58 @@ def _get_redis_client_logic(**env_overrides):
     if _service_name is not None:
         redis_kwargs["service_name"] = _service_name
 
-    # Handle GCP IAM authentication
-    _gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str("REDIS_GCP_SERVICE_ACCOUNT")
-    _gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
-
-    if _gcp_service_account is not None:
-        verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
-        redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
-            service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
+    if redis_kwargs.get("credential_provider") is None:
+        # Handle GCP IAM authentication
+        _gcp_service_account: Final = redis_kwargs.get("gcp_service_account") or get_secret_str(
+            "REDIS_GCP_SERVICE_ACCOUNT"
         )
-        # Store GCP service account in redis_connect_func for async cluster access
-        redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
+        _gcp_ssl_ca_certs: Final = redis_kwargs.get("gcp_ssl_ca_certs") or get_secret_str("REDIS_GCP_SSL_CA_CERTS")
 
-        # Remove GCP-specific kwargs that shouldn't be passed to Redis client
-        redis_kwargs.pop("gcp_service_account", None)
-        redis_kwargs.pop("gcp_ssl_ca_certs", None)
+        if _gcp_service_account is not None:
+            verbose_logger.debug("Setting up GCP IAM authentication for Redis with service account.")
+            redis_kwargs["redis_connect_func"] = create_gcp_iam_redis_connect_func(
+                service_account=_gcp_service_account, ssl_ca_certs=_gcp_ssl_ca_certs
+            )
+            # Store GCP service account in redis_connect_func for async cluster access
+            redis_kwargs["redis_connect_func"]._gcp_service_account = _gcp_service_account
 
-        # Only enable SSL if explicitly requested AND SSL CA certs are provided
-        if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
-            redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
+            # Only enable SSL if explicitly requested AND SSL CA certs are provided
+            if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
+                redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
 
-    # Handle Azure AD authentication (after GCP IAM block)
-    _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
+        # Handle Azure AD authentication (after GCP IAM block)
+        _azure_redis_ad_token: Final = redis_kwargs.get("azure_redis_ad_token") or get_secret("REDIS_AZURE_AD_TOKEN")
 
-    _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
+        _azure_ad_enabled: Final = _azure_redis_ad_token is not None and str(_azure_redis_ad_token).lower() == "true"
 
-    if _azure_ad_enabled and _gcp_service_account is not None:
-        verbose_logger.warning(
-            "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
-            "Using GCP IAM. Remove one to avoid misconfiguration."
-        )
+        if _azure_ad_enabled and _gcp_service_account is not None:
+            verbose_logger.warning(
+                "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
+                "Using GCP IAM. Remove one to avoid misconfiguration."
+            )
 
-    if _azure_ad_enabled and _gcp_service_account is None:
-        _azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
-        _azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
-        _azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str("AZURE_CLIENT_SECRET")
+        if _azure_ad_enabled and _gcp_service_account is None:
+            _azure_client_id: Final = redis_kwargs.get("azure_client_id") or get_secret_str("AZURE_CLIENT_ID")
+            _azure_tenant_id: Final = redis_kwargs.get("azure_tenant_id") or get_secret_str("AZURE_TENANT_ID")
+            _azure_client_secret: Final = redis_kwargs.get("azure_client_secret") or get_secret_str(
+                "AZURE_CLIENT_SECRET"
+            )
 
-        verbose_logger.debug("Setting up Azure AD authentication for Redis.")
-        redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
-            azure_client_id=_azure_client_id,
-            azure_tenant_id=_azure_tenant_id,
-            azure_client_secret=_azure_client_secret,
-        )
-        # Marker for async paths to detect Azure AD auth. The live credential
-        # object is attached separately as `_azure_credential` by
-        # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
-        # are intentionally NOT exposed on the function to avoid leaking
-        # credentials via inspection or logging.
-        redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
+            verbose_logger.debug("Setting up Azure AD authentication for Redis.")
+            redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
+                azure_client_id=_azure_client_id,
+                azure_tenant_id=_azure_tenant_id,
+                azure_client_secret=_azure_client_secret,
+            )
+            # Marker for async paths to detect Azure AD auth. The live credential
+            # object is attached separately as `_azure_credential` by
+            # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
+            # are intentionally NOT exposed on the function to avoid leaking
+            # credentials via inspection or logging.
+            redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True
+
+    redis_kwargs.pop("gcp_service_account", None)
+    redis_kwargs.pop("gcp_ssl_ca_certs", None)
 
     # Always remove Azure-specific kwargs that shouldn't be passed to Redis client
     redis_kwargs.pop("azure_redis_ad_token", None)
@@ -465,6 +478,13 @@ def _get_redis_client_logic(**env_overrides):
     redis_kwargs.pop("azure_tenant_id", None)
     redis_kwargs.pop("azure_client_secret", None)
 
+    if redis_kwargs.get("credential_provider") is not None:
+        redis_kwargs.pop("redis_connect_func", None)
+        redis_kwargs.pop("username", None)
+        redis_kwargs.pop("password", None)
+        if redis_kwargs.get("url") is not None:
+            redis_kwargs["url"] = _url_without_userinfo(redis_kwargs["url"])
+
     if "url" in redis_kwargs and redis_kwargs["url"] is not None:
         # Only strip host/port/db/password when not routing to a cluster.
         # When startup_nodes is also present the cluster path takes priority and
@@ -532,8 +552,7 @@ def _init_redis_sentinel(redis_kwargs) -> redis.Redis:
     service_name: Final = redis_kwargs.get("service_name")
     connection_kwargs: Final = _get_redis_sentinel_connection_kwargs(redis_kwargs)
     connection_kwargs.setdefault("socket_timeout", REDIS_SOCKET_TIMEOUT)
-    sentinel_kwargs: Final = dict(connection_kwargs)
-    sentinel_kwargs["password"] = sentinel_password
+    sentinel_kwargs: Final = _sentinel_auth_kwargs(connection_kwargs, sentinel_password)
 
     if not sentinel_nodes or not service_name:
         raise ValueError("Both 'sentinel_nodes' and 'service_name' are required for Redis Sentinel.")
@@ -605,7 +624,12 @@ def _async_credential_provider(redis_connect_func: object | None) -> CredentialP
 def _async_auth_kwargs(redis_kwargs: dict) -> dict:
     """Swaps a connect func an async path cannot run for the equivalent credential provider,
     which supersedes any static username or password redis-py would otherwise reject it with."""
-    credential_provider: Final = _async_credential_provider(redis_kwargs.get("redis_connect_func"))
+    explicit_provider: Final = redis_kwargs.get("credential_provider")
+    credential_provider: Final = (
+        explicit_provider
+        if explicit_provider is not None
+        else _async_credential_provider(redis_kwargs.get("redis_connect_func"))
+    )
     if credential_provider is None:
         return redis_kwargs
 
@@ -738,8 +762,20 @@ def get_redis_connection_pool(
     return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)
 
 
+def _redis_kwargs_for_logging(redis_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    return {
+        key: "<credential provider>"
+        if key == "credential_provider" and value is not None
+        else "<redis connect function>"
+        if key == "redis_connect_func" and value is not None
+        else value
+        for key, value in redis_kwargs.items()
+    }
+
+
 def _pretty_print_redis_config(redis_kwargs: dict) -> None:
     """Pretty print the Redis configuration using rich with sensitive data masking"""
+    redis_kwargs_for_logging: Final = _redis_kwargs_for_logging(redis_kwargs)
     try:
         import logging
 
@@ -757,7 +793,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
         masker = SensitiveDataMasker()
 
         # Mask sensitive data in redis_kwargs
-        masked_redis_kwargs = masker.mask_dict(redis_kwargs)
+        masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
 
         # Create main panel title
         title: Final = Text("Redis Configuration", style="bold blue")
@@ -820,7 +856,7 @@ def _pretty_print_redis_config(redis_kwargs: dict) -> None:
     except ImportError:
         # Fallback to simple logging if rich is not available
         masker = SensitiveDataMasker()
-        masked_redis_kwargs = masker.mask_dict(redis_kwargs)
+        masked_redis_kwargs = masker.mask_dict(redis_kwargs_for_logging)
         verbose_logger.info("Redis configuration: %s", masked_redis_kwargs)
     except Exception as e:
         verbose_logger.error("Error pretty printing Redis configuration: %s", e)

litellm/a2a_protocol/card_resolver.py

@@ -4,6 +4,7 @@ Custom A2A Card Resolver for LiteLLM.
 Extends the A2A SDK's card resolver to support multiple well-known paths.
 """
 
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_logger
@@ -48,6 +49,43 @@ def is_localhost_or_internal_url(url: str | None) -> bool:
     return any(pattern in url_lower for pattern in LOCALHOST_URL_PATTERNS)
 
 
+_CANONICAL_PROTOCOL_BINDINGS: Final = MappingProxyType(
+    {
+        "jsonrpc": "JSONRPC",
+        "http+json": "HTTP+JSON",
+        "grpc": "GRPC",
+    }
+)
+
+_LEGACY_PROTOCOL_VERSION: Final = "0.3"
+
+
+def normalize_agent_card_interfaces(agent_card: "AgentCard") -> "AgentCard":
+    """
+    Canonicalize the supported interfaces of spec-adjacent agent cards.
+
+    Some A2A servers (e.g. LangGraph Platform) serve agent cards with lowercase
+    bindings like "jsonrpc", but a2a-sdk's ClientFactory matches bindings
+    case-sensitively against its uppercase TransportProtocol constants and fails
+    with "no compatible transports found." for spec-adjacent casings.
+
+    The same servers also speak the A2A 0.3 JSON dialect ("kind"-discriminated
+    payloads) while declaring protocolVersion "1.0", which a2a-sdk's strict v1
+    proto parsing rejects. A mis-cased binding fingerprints such a server, so its
+    declared version is downgraded to 0.3 to route the SDK's ClientFactory onto
+    its v0.3 compat transport, which speaks that dialect.
+    """
+    normalized: Final = type(agent_card)()
+    normalized.CopyFrom(agent_card)
+    for interface in normalized.supported_interfaces:
+        canonical: str | None = _CANONICAL_PROTOCOL_BINDINGS.get(interface.protocol_binding.lower())
+        if canonical is None or canonical == interface.protocol_binding:
+            continue
+        interface.protocol_binding = canonical
+        interface.protocol_version = _LEGACY_PROTOCOL_VERSION
+    return normalized
+
+
 def get_agent_card_url(agent_card: "AgentCard") -> str | None:
     """Return the agent endpoint URL from the resolved SDK card."""
     url: Final = getattr(agent_card, "url", None)

litellm/a2a_protocol/main.py

@@ -73,6 +73,7 @@ except ImportError:
 from litellm.a2a_protocol.card_resolver import (
     LiteLLMA2ACardResolver,
     get_agent_card_url,
+    normalize_agent_card_interfaces,
 )
 from litellm.a2a_protocol.exception_mapping_utils import (
     handle_a2a_localhost_retry,
@@ -782,13 +783,17 @@ async def create_a2a_client(
     if extra_headers:
         verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
 
+    resolver: Final = A2ACardResolver(httpx_client=httpx_client, base_url=base_url)
+    agent_card: Final = normalize_agent_card_interfaces(
+        await resolver.get_agent_card(http_kwargs={"headers": extra_headers} if extra_headers else None)
+    )
+
     a2a_client: Final = await create_client(  # pyright: ignore[reportOptionalCall]
-        base_url,
+        agent_card,
         client_config=ClientConfig(  # pyright: ignore[reportOptionalCall]
             httpx_client=httpx_client,
             streaming=streaming,
         ),
-        resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
     )
     # Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
     # the configured httpx client and this agent's headers without excavating
@@ -799,9 +804,7 @@ async def create_a2a_client(
         if extra_headers
         else None
     )
-    agent_card: Final = getattr(a2a_client, "_card", None)
-    if agent_card is not None:
-        a2a_client._litellm_agent_card = agent_card
+    a2a_client._litellm_agent_card = agent_card
 
     verbose_logger.info("A2A client created for %s", base_url)
 

litellm/batches/batch_utils.py

@@ -551,7 +551,7 @@ def _get_batch_job_usage_from_response_body(
     return usage
 
 
-def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict:
+def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> Mapping[str, Any]:
     """
     Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
 
@@ -563,7 +563,7 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[st
 
 def _get_response_from_batch_job_output_file(
     batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
-) -> Any:
+) -> Mapping[str, Any]:
     """
     Get the response from the batch job output file
     """

litellm/caching/caching_handler.py

@@ -18,7 +18,7 @@ import asyncio
 import datetime
 import inspect
 import time
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping
+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
 
 from pydantic import BaseModel
@@ -27,6 +27,7 @@ import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.caching import InMemoryCache
 from litellm.caching.caching import S3Cache
+from litellm.constants import CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
     update_response_metadata,
 )
@@ -124,6 +125,29 @@ def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") ->
     return details.model_dump(exclude_none=True) if hasattr(details, "model_dump") else {}
 
 
+_PENDING_CACHE_WRITES: Final[set["asyncio.Task[None]"]] = set()  # mutable-ok: strong refs to pending write tasks
+
+
+async def _complete_cache_write_despite_cancellation(write_factory: Callable[[], Awaitable[None]]) -> None:
+    try:
+        await write_factory()
+    except asyncio.CancelledError:
+        try:
+            await asyncio.wait_for(write_factory(), timeout=CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS)
+        except Exception as flush_error:  # noqa: BLE001  # shutdown flush failures are logged, never raised
+            verbose_logger.warning(
+                "LiteLLM Cache: pending cache write failed during event loop shutdown: %s", flush_error
+            )
+        raise
+
+
+def create_cache_write_task(write_factory: Callable[[], Awaitable[None]]) -> "asyncio.Task[None]":
+    task: Final = asyncio.create_task(_complete_cache_write_despite_cancellation(write_factory))
+    _PENDING_CACHE_WRITES.add(task)
+    task.add_done_callback(_PENDING_CACHE_WRITES.discard)
+    return task
+
+
 def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None:
     """Read the caller-supplied ``cache_key`` off the request kwargs."""
     return request_kwargs.get("cache_key", None)
@@ -983,6 +1007,7 @@ class LLMCachingHandler:
 
         if litellm.cache is None:
             return
+        cache: Final = litellm.cache
 
         new_kwargs: Final = kwargs.copy()
         new_kwargs.update(
@@ -1004,24 +1029,24 @@ class LLMCachingHandler:
             ):
                 if (
                     isinstance(result, EmbeddingResponse)
-                    and litellm.cache is not None
-                    and not isinstance(litellm.cache.cache, S3Cache)  # s3 doesn't support bulk writing. Exclude.
+                    and not isinstance(cache.cache, S3Cache)  # s3 doesn't support bulk writing. Exclude.
                 ):
-                    asyncio.create_task(
-                        litellm.cache.async_add_cache_pipeline(
+                    create_cache_write_task(
+                        lambda: cache.async_add_cache_pipeline(
                             result, dynamic_cache_object=self.dual_cache, **new_kwargs
                         )
                     )
                 else:
-                    asyncio.create_task(
-                        litellm.cache.async_add_cache(
-                            result.model_dump_json(),
+                    result_json: Final = result.model_dump_json()
+                    create_cache_write_task(
+                        lambda: cache.async_add_cache(
+                            result_json,
                             dynamic_cache_object=self.dual_cache,
                             **new_kwargs,
                         )
                     )
             else:
-                asyncio.create_task(litellm.cache.async_add_cache(result, **new_kwargs))
+                create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
 
     def sync_set_cache(
         self,

litellm/caching/redis_cache.py

@@ -175,6 +175,10 @@ _RedisCallResult = TypeVar("_RedisCallResult")
 _swallowed_redis_failures: Final[ContextVar[int]] = ContextVar("litellm_swallowed_redis_failures", default=0)
 
 
+def _opaque_kwarg_key(value: object) -> str:
+    return f"{type(value).__name__}-{id(value)}"
+
+
 @functools.lru_cache(maxsize=1)
 def _redis_health_error_types() -> tuple[type, ...]:
     """Exception types that mean the Redis backend itself is unhealthy.
@@ -399,10 +403,9 @@ class RedisCache(BaseCache):
         Generate a cache key for the async Redis client based on connection parameters.
         This ensures different Redis configurations use different cached clients.
         """
-        # Create a stable representation of redis_kwargs for hashing
         # Sort keys to ensure consistent hash regardless of parameter order
         sorted_kwargs: Final = sorted(self.redis_kwargs.items())
-        kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True)
+        kwargs_str: Final = json.dumps(sorted_kwargs, sort_keys=True, default=_opaque_kwarg_key)
         kwargs_hash: Final = hashlib.sha256(kwargs_str.encode()).hexdigest()[:16]
         return f"async-redis-client-{kwargs_hash}"
 
@@ -432,7 +435,7 @@ class RedisCache(BaseCache):
         """
         if key is None:
             return key
-        if self.namespace is not None and not key.startswith(self.namespace):
+        if self.namespace and not key.startswith(self.namespace + ":"):
             key = self.namespace + ":" + key
 
         return key
@@ -1384,10 +1387,10 @@ class RedisCache(BaseCache):
             dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
         """
         try:
-            import redis.asyncio as redis_async
+            from .._redis import get_redis_async_client
 
             # Create a fresh Redis client with current settings
-            redis_client: Final = redis_async.Redis(**self.redis_kwargs)
+            redis_client: Final = get_redis_async_client(**self.redis_kwargs)
 
             # Test the connection
             ping_result: Final = await redis_client.ping()

litellm/caching/redis_cluster_cache.py

@@ -64,22 +64,9 @@ class RedisClusterCache(RedisCache):
             dict: {"status": "success" | "failed", "message": str, "error": Optional[str]}
         """
         try:
-            import redis.asyncio as redis_async
-            from redis.cluster import ClusterNode
+            from .._redis import get_redis_async_client
 
-            # Create ClusterNode objects from startup_nodes
-            cluster_kwargs: Final = self.redis_kwargs.copy()
-            startup_nodes: Final = cluster_kwargs.pop("startup_nodes", [])
-
-            new_startup_nodes: Final[list[ClusterNode]] = []
-            for item in startup_nodes:
-                new_startup_nodes.append(ClusterNode(**item))
-
-            # Create a fresh Redis Cluster client with current settings
-            redis_client: Final = redis_async.RedisCluster(
-                startup_nodes=new_startup_nodes,
-                **cluster_kwargs,
-            )
+            redis_client: Final = get_redis_async_client(**self.redis_kwargs)
 
             # Test the connection
             ping_result: Final = await redis_client.ping()

litellm/caching/redis_cluster_node_isolation.py

@@ -18,6 +18,14 @@ already does when one of its pooled connections errors), leaving every other nod
 connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
+
+redis-py 8.x fixed this upstream with gentler machinery than this override's
+``node.disconnect()`` (which also kills connections other coroutines are mid-operation
+on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
+killed connection): it marks in-use connections for reconnect only after their current
+operation completes, disconnects only the idle pooled ones, and defers reinitialization
+to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
+recovery API, the factory returns the base ``RedisCluster`` unmodified.
 """
 
 import asyncio
@@ -72,8 +80,16 @@ class _ClusterAttrs(Protocol):
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
 
 
-def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
-    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+def get_litellm_async_redis_cluster_class(
+    cluster_node_class: type | None = None,
+) -> type["_AsyncRedisClusterType"]:
+    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
+    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch
+    tears down the whole cluster client.
+
+    ``cluster_node_class`` exists for dependency injection in tests; production callers
+    leave it unset and the installed ``ClusterNode`` is used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -81,7 +97,10 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     """
     import redis
     from redis.asyncio.cluster import (
-        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+        ClusterNode as _AsyncClusterNode,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # same stale-stub gap as the import above
     )
     from redis.cluster import get_node_name
     from redis.commands import READ_COMMANDS
@@ -98,6 +117,15 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     from redis.exceptions import ConnectionError as _RedisConnectionError
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
+    node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    if hasattr(node_class, "update_active_connections_for_reconnect"):
+        verbose_logger.debug(
+            "redis-py %s recovers a node-level connection error per-connection upstream; "
+            "using the base RedisCluster without litellm's node-isolation override.",
+            redis.__version__,
+        )
+        return _BaseAsyncRedisCluster
+
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(
             "redis-py %s is not in the set this cluster-teardown-storm fix was verified "

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -21,6 +21,9 @@ from pydantic import BaseModel
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    responses_reasoning_item_from_thinking_blocks,
+)
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.bridges.completion_transformation import (
     CompletionTransformationBridge,
@@ -32,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import normalize_responses_api_stream_options
 from litellm.types.llms.openai import (
+    REASONING_EFFORT,
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
     ChatCompletionToolCallChunk,
@@ -55,9 +59,11 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import (
         ALL_RESPONSES_API_TOOL_PARAMS,
         AllMessageValues,
+        ChatCompletionFileObject,
         ChatCompletionImageObject,
         ChatCompletionRedactedThinkingBlock,
         ChatCompletionThinkingBlock,
+        ChatCompletionToolReferenceObject,
         OpenAIMessageContentListBlock,
     )
     from litellm.types.utils import Choices
@@ -85,6 +91,22 @@ def _get_reasoning_items(
     return []
 
 
+def _reasoning_input_items(msg: "AllMessageValues") -> list[dict[str, object]]:  # mutable-ok: API message payload
+    """Reasoning input items for an assistant message.
+
+    Stored reasoning items win because they carry an id the Responses API minted; thinking
+    blocks are the fallback for turns that arrived over another API surface.
+    """
+    items: Final = _get_reasoning_items(msg)
+    stored: Final = [_reasoning_item_to_response_input(item) for item in items]  # mutable-ok: API message payload
+    if stored:
+        return stored
+    raw_blocks: Final = msg.get("thinking_blocks") or ()
+    blocks: Final = cast("Iterable[ChatCompletionThinkingBlock]", raw_blocks)  # cast-ok: untyped client json
+    from_thinking: Final = responses_reasoning_item_from_thinking_blocks(blocks)
+    return [] if from_thinking is None else [dict(from_thinking)]  # mutable-ok: API message payload
+
+
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: str | None,
@@ -155,6 +177,16 @@ def _map_incomplete_reason_to_finish_reason(incomplete_reason: str | None) -> Li
     return "length"
 
 
+def _input_file_from_file_value(file_value: object) -> dict[str, object]:
+    if not isinstance(file_value, dict):
+        return {"type": "input_file"}
+    file_dict: Final = cast("dict[str, object]", file_value)  # cast-ok: runtime dict checked
+    return {
+        "type": "input_file",
+        **{key: file_dict[key] for key in ("file_id", "file_data", "filename") if key in file_dict},
+    }
+
+
 def _incomplete_reason_from_response_payload(response_payload: object) -> str | None:
     if not isinstance(response_payload, Mapping):
         return None
@@ -372,8 +404,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         )
                     )
             elif role == "assistant" and tool_calls and isinstance(tool_calls, list):
-                for r_item in _get_reasoning_items(msg):
-                    input_items.append(_reasoning_item_to_response_input(r_item))
+                input_items.extend(_reasoning_input_items(msg))
+                if content:
+                    input_items.append(
+                        {  # mutable-ok: API message payload
+                            "type": "message",
+                            "role": "assistant",
+                            "content": self._convert_content_to_responses_format(content, "assistant"),
+                        }
+                    )
                 for tool_call in tool_calls:
                     function = tool_call.get("function")
                     custom = tool_call.get("custom")
@@ -400,15 +439,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         raise ValueError(f"tool call not supported: {tool_call}")
             elif content is not None:
                 if role == "assistant":
-                    for r_item in _get_reasoning_items(msg):
-                        input_items.append(_reasoning_item_to_response_input(r_item))
+                    input_items.extend(_reasoning_input_items(msg))
                 input_items.append(
-                    {
+                    {  # mutable-ok: API message payload
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(content, cast(str, role)),
                     }
                 )
+            elif role == "assistant":
+                input_items.extend(_reasoning_input_items(msg))
 
         return input_items, instructions
 
@@ -929,7 +969,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         content: str
         | list[object]
         | Iterable[
-            Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
+            Union[
+                "OpenAIMessageContentListBlock",
+                "ChatCompletionThinkingBlock",
+                "ChatCompletionRedactedThinkingBlock",
+                "ChatCompletionToolReferenceObject",
+            ]
         ]
         | None,
         role: str,
@@ -978,17 +1023,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   image -> %s", converted)
                         elif item_type == "file":
-                            # Map Chat Completion file to Responses API input_file
-                            # {"type": "file", "file": {"file_data": "...", "filename": "..."}}
-                            # -> {"type": "input_file", "file_data": "...", "filename": "..."}
-                            file_data = item.get("file", {})
-                            converted = {"type": "input_file"}
-                            if isinstance(file_data, dict):
-                                for key in ["file_id", "file_data", "filename"]:
-                                    if key in file_data:
-                                        converted[key] = file_data[key]
+                            converted = _input_file_from_file_value(
+                                cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
+                            )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
+                        elif item_type == "tool_reference":
+                            verbose_logger.debug(
+                                "Chat provider:   tool_reference has no responses API equivalent; skipped"
+                            )
                         elif item_type in [
                             "input_text",
                             "input_image",
@@ -1086,22 +1129,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # If string is passed, map with optional summary based on flag/env var
-        if reasoning_effort == "none":
-            return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
-        elif reasoning_effort == "high":
-            return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
-        elif reasoning_effort == "xhigh":
-            return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
-        elif reasoning_effort == "medium":
+        if reasoning_effort in get_args(REASONING_EFFORT):
             return (
-                Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
-            )
-        elif reasoning_effort == "low":
-            return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
-        elif reasoning_effort == "minimal":
-            return (
-                Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
+                Reasoning(effort=reasoning_effort, summary="detailed")
+                if auto_summary_enabled
+                else Reasoning(effort=reasoning_effort)
             )
         return None
 

litellm/constants.py

@@ -48,6 +48,9 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Data URIs exceeding this are replaced with a size placeholder.
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
+REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
+# in-memory stand-in handed to provider converters for redacted arguments; never stored
+REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
 
 MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
 
@@ -146,6 +149,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-adaptive-router-model",
     "x-litellm-applied-guardrails",
     "x-litellm-guardrail-scan-id",
+    "x-litellm-cache-key",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -292,6 +296,9 @@ GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int(
     os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60)
 )
 BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000
+DEFAULT_PRESIDIO_ANALYZE_CHUNK_SIZE_BYTES: Final = 500_000
+PRESIDIO_ANALYZE_CHUNK_OVERLAP_CHARS: Final = 4096
+PRESIDIO_ANALYZE_CHUNK_CONCURRENCY: Final = 8
 # Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger.
 # Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire.
 MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))
@@ -377,6 +384,7 @@ AZURE_OPERATION_POLLING_TIMEOUT: Final = int(os.getenv("AZURE_OPERATION_POLLING_
 AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: Final = str(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2024-11-30"))
 AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: Final = int(os.getenv("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", 96))
 REDIS_SOCKET_TIMEOUT: Final = float(os.getenv("REDIS_SOCKET_TIMEOUT", 0.1))
+CACHE_WRITE_SHUTDOWN_FLUSH_TIMEOUT_SECONDS: Final[float] = 5.0
 REDIS_CONNECTION_POOL_TIMEOUT: Final = int(os.getenv("REDIS_CONNECTION_POOL_TIMEOUT", 5))
 REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_FAILURE_THRESHOLD", 5))
 REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT: Final = int(os.getenv("REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT", 60))
@@ -460,6 +468,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
 ]
 STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
+STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
+STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
     os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
@@ -749,6 +759,7 @@ openai_compatible_endpoints: Final[list] = [
     "api.groq.com/openai/v1",
     "https://integrate.api.nvidia.com/v1",
     "api.deepseek.com/v1",
+    "api.together.ai/v1",
     "api.together.xyz/v1",
     "app.empower.dev/api/v1",
     "https://api.friendli.ai/serverless/v1",
@@ -1356,8 +1367,6 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
 LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
-AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
-ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@@ -1467,6 +1476,12 @@ LITELLM_PROXY_MASTER_KEY_ALIAS: Final = "litellm_proxy_master_key"
 # ``ProxyLogging._handle_logging_proxy_only_error``.
 LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL: Final = "litellm_no_upstream_llm_call"
 
+# Key/team metadata fields naming the OTel Resource ``service.name``, highest
+# precedence first. Shared between the OTel v2 tenant router (which reads them
+# out of ``user_api_key_auth_metadata``) and proxy request setup (which re-applies
+# the key's values after the team metadata merge so a key outranks its team).
+OTEL_SERVICE_NAME_METADATA_KEYS: Final = ("otel_service_name_override", "otel_service_name")
+
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED: Final = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")
 LITELLM_KEY_ROTATION_CHECK_INTERVAL_SECONDS: Final = int(
@@ -1542,6 +1557,8 @@ SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BA
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
+RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS: Final = max(1, int(os.getenv("RESPONSES_SESSION_LOOKUP_MAX_ATTEMPTS", "3")))
+RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL: Final = float(os.getenv("RESPONSES_SESSION_LOOKUP_RETRY_INTERVAL", "0.2"))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60))  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
@@ -1561,6 +1578,19 @@ STALE_OBJECT_CLEANUP_BATCH_SIZE: Final = max(1, int(os.getenv("STALE_OBJECT_CLEA
 # installations with large numbers of stale managed objects).
 _batch_polling_env: Final = os.getenv("PROXY_BATCH_POLLING_ENABLED", "true").lower()
 PROXY_BATCH_POLLING_ENABLED: Final = _batch_polling_env == "true"
+BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS", "5")
+)
+BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS", "60")
+)
+BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS: Final = float(
+    os.getenv("BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS", "3600")
+)
+_background_interaction_cost_polling_env: Final = os.getenv(
+    "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
+).lower()
+BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
 PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605))
 PROXY_BATCH_WRITE_AT: Final = int(os.getenv("PROXY_BATCH_WRITE_AT", 10))  # in seconds, increased from 10
 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)
@@ -1625,6 +1655,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
     "enable_anthropic_prompt_caching",
     "anthropic_prompt_caching_ttl",
     "max_ui_session_budget",
+    "budget_rollover",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))
@@ -1791,6 +1822,43 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
 
 UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
 
+# A retrieved response replays the usage of the call that created it, so pricing these
+# read/management routes like inference bills the same tokens twice.
+NON_INFERENCE_CALL_TYPES: Final[frozenset[str]] = frozenset(
+    {
+        "get_responses",
+        "aget_responses",
+        "delete_responses",
+        "adelete_responses",
+        "cancel_responses",
+        "acancel_responses",
+        "list_input_items",
+        "alist_input_items",
+        "vector_store_create",
+        "avector_store_create",
+        "vector_store_retrieve",
+        "avector_store_retrieve",
+        "vector_store_list",
+        "avector_store_list",
+        "vector_store_update",
+        "avector_store_update",
+        "vector_store_delete",
+        "avector_store_delete",
+        "vector_store_file_create",
+        "avector_store_file_create",
+        "vector_store_file_list",
+        "avector_store_file_list",
+        "vector_store_file_retrieve",
+        "avector_store_file_retrieve",
+        "vector_store_file_content",
+        "avector_store_file_content",
+        "vector_store_file_update",
+        "avector_store_file_update",
+        "vector_store_file_delete",
+        "avector_store_file_delete",
+    }
+)
+
 # PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this
 # sentinel api_key so PTU flat cost stays distinguishable from real per-request
 # spend under the table's composite unique constraint.

litellm/cost_calculator.py

@@ -2,6 +2,7 @@
 ## File for 'response_cost' calculation in Logging
 import logging
 import time
+from collections.abc import Sequence
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
@@ -19,6 +20,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
 from litellm.litellm_core_utils.llm_cost_calc.usage_object_transformation import (
+    InteractionsUsageObjectTransformation,
     TranscriptionUsageObjectTransformation,
 )
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -74,7 +76,10 @@ from litellm.llms.perplexity.cost_calculator import (
 from litellm.llms.tencent.cost_calculator import (
     cost_per_token as tencent_cost_per_token,
 )
-from litellm.llms.together_ai.cost_calculator import get_model_params_and_category
+from litellm.llms.together_ai.cost_calculator import (
+    get_model_params_and_category,
+    has_together_registry_pricing,
+)
 from litellm.llms.vertex_ai.cost_calculator import (
     cost_per_character as google_cost_per_character,
 )
@@ -150,6 +155,7 @@ _VIDEO_CALL_TYPES: Final = frozenset(
     }
 )
 
+
 _SPEECH_CALL_TYPES: Final = frozenset(
     {
         CallTypes.speech.value,
@@ -554,9 +560,10 @@ def cost_per_token(
         )
     elif call_type == "atranscription" or call_type == "transcription":
         if _transcription_usage_has_token_details(usage_block):
-            return openai_cost_per_token(
+            return generic_cost_per_token(
                 model=model_without_prefix,
                 usage=usage_block,
+                custom_llm_provider=custom_llm_provider,
                 service_tier=service_tier,
                 data_residency=data_residency,
             )
@@ -589,6 +596,7 @@ def cost_per_token(
                 prompt_characters=prompt_characters,
                 completion_characters=completion_characters,
                 usage=usage_block,
+                service_tier=service_tier,
                 vertex_location=vertex_location,
             )
         elif cost_router == "cost_per_token":
@@ -792,14 +800,27 @@ def _select_model_name_for_cost_calc(
         and custom_llm_provider is not None
         and not _model_contains_known_llm_provider(return_model)
     ):  # add provider prefix if not already present, to match model_cost
-        if region_name is not None:
-            return_model = f"{custom_llm_provider}/{region_name}/{return_model}"
-        else:
-            return_model = f"{custom_llm_provider}/{return_model}"
+        provider_prefix: Final = custom_llm_provider if region_name is None else f"{custom_llm_provider}/{region_name}"
+        return_model = _strip_unregistered_leading_segments(f"{provider_prefix}/{return_model}", region_name)
 
     return return_model
 
 
+def _strip_unregistered_leading_segments(model: str, region_name: str | None) -> str:
+    """Resolve a provider-prefixed slash alias like "vertex_ai/vertex/claude-opus-5" to the
+    registered cost key ("vertex_ai/claude-opus-5"), keeping the model unchanged when it already
+    resolves downstream (custom-priced router ids) or no stripped candidate is registered (#38069)."""
+    segments: Final = model.split("/")
+    if "/".join(segments[1:]) in litellm.model_cost:
+        return model
+    head_len: Final = 2 if region_name is not None and len(segments) > 2 and segments[1] == region_name else 1
+    head: Final = "/".join(segments[:head_len])
+    tail: Final = segments[head_len:]
+    strippable: Final = next((index for index, segment in enumerate(tail) if segment in LlmProvidersSet), len(tail))
+    candidates: Final = (f"{head}/{'/'.join(tail[start:])}" for start in range(min(strippable, len(tail) - 1) + 1))
+    return next((candidate for candidate in candidates if candidate in litellm.model_cost), model)
+
+
 @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
 def _model_contains_known_llm_provider(model: str) -> bool:
     """
@@ -830,9 +851,11 @@ def _get_response_model(completion_response: object) -> str | None:
 _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
     # ON_DEMAND_PRIORITY maps to "priority" — selects input_cost_per_token_priority, etc.
     "ON_DEMAND_PRIORITY": "priority",
-    # FLEX / BATCH maps to "flex" — selects input_cost_per_token_flex, etc.
+    # FLEX / BATCH / ON_DEMAND_FLEX maps to "flex" — selects input_cost_per_token_flex, etc.
+    # Vertex AI reports flex/shared-capacity traffic as ON_DEMAND_FLEX, not FLEX.
     "FLEX": "flex",
     "BATCH": "flex",
+    "ON_DEMAND_FLEX": "flex",
     # ON_DEMAND is standard pricing — no service_tier suffix applied
     "ON_DEMAND": None,
 }
@@ -847,9 +870,9 @@ def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
 
     trafficType values seen in practice
     ------------------------------------
-    ON_DEMAND          -> standard pricing  (service_tier = None)
-    ON_DEMAND_PRIORITY -> priority pricing  (service_tier = "priority")
-    FLEX / BATCH       -> batch/flex pricing (service_tier = "flex")
+    ON_DEMAND               -> standard pricing  (service_tier = None)
+    ON_DEMAND_PRIORITY      -> priority pricing  (service_tier = "priority")
+    FLEX / BATCH / ON_DEMAND_FLEX -> batch/flex pricing (service_tier = "flex")
     """
     if traffic_type is None:
         return None
@@ -912,6 +935,8 @@ def _get_usage_object(
                 usage_obj,
             )
         )
+    elif isinstance(usage_obj, dict) and InteractionsUsageObjectTransformation.is_interactions_usage_object(usage_obj):
+        return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage_obj)
     elif isinstance(usage_obj, dict):
         return Usage(**usage_obj)
     elif isinstance(usage_obj, BaseModel):
@@ -1288,6 +1313,10 @@ def completion_cost(
                         )
                         if tr_usage is not None:
                             _usage = tr_usage.model_dump()
+                    elif InteractionsUsageObjectTransformation.is_interactions_usage_object(_usage):
+                        _usage = InteractionsUsageObjectTransformation.transform_interactions_usage_object(
+                            _usage
+                        ).model_dump()
                     else:
                         _usage = _usage
 
@@ -1372,23 +1401,36 @@ def completion_cost(
                     if custom_pricing and litellm_logging_obj is not None:
                         _litellm_params = getattr(litellm_logging_obj, "litellm_params", None)
                         if _litellm_params is not None:
-                            _metadata = _litellm_params.get("metadata", {}) or {}
-                            _video_model_info = _metadata.get("model_info", None)
+                            _video_model_info = next(
+                                (
+                                    model_info
+                                    for _metadata_key in ("metadata", "litellm_metadata")
+                                    if (model_info := (_litellm_params.get(_metadata_key) or {}).get("model_info"))
+                                    is not None
+                                ),
+                                None,
+                            )
 
                     usage_obj = getattr(completion_response, "usage", None)
                     duration_seconds: float | None = None
                     video_resolution: str | None = None
+                    provider_reported_cost: float | None = None
                     if completion_response is not None and usage_obj:
                         # Handle both dict and Pydantic Usage object
                         if isinstance(usage_obj, dict):
                             duration_seconds = usage_obj.get("duration_seconds", None)
                             _vr = usage_obj.get("video_resolution", None)
+                            provider_reported_cost = usage_obj.get("provider_reported_cost_usd", None)
                         else:
                             duration_seconds = getattr(usage_obj, "duration_seconds", None)
                             _vr = getattr(usage_obj, "video_resolution", None)
+                            provider_reported_cost = getattr(usage_obj, "provider_reported_cost_usd", None)
                         if _vr is not None:
                             video_resolution = str(_vr).strip().lower()
 
+                        if _video_model_info is None and provider_reported_cost is not None:
+                            return float(provider_reported_cost)
+
                         if duration_seconds is not None:
                             # Calculate cost based on video duration using video-specific cost calculation
                             from litellm.llms.openai.cost_calculation import (
@@ -1530,10 +1572,9 @@ def completion_cost(
 
                     return MCPCostCalculator.calculate_mcp_tool_call_cost(litellm_logging_obj=litellm_logging_obj)
                 # Calculate cost based on prompt_tokens, completion_tokens
-                if "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai":
-                    # together ai prices based on size of llm
-                    # get_model_params_and_category takes a model name and returns the category of LLM size it is in model_prices_and_context_window.json
-
+                if (
+                    "togethercomputer" in model or "together_ai" in model or custom_llm_provider == "together_ai"
+                ) and not has_together_registry_pricing(model, litellm.model_cost):
                     model = get_model_params_and_category(model, call_type=CallTypes(call_type))
 
                 # replicate llms are calculate based on time for request running
@@ -2336,6 +2377,64 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
 
 
+def _candidate_realtime_token_costs(
+    model_name: str,
+    combined_usage_object: Usage,
+    custom_llm_provider: str,
+    data_residency: str | None,
+) -> tuple[float, float] | None:
+    try:
+        return generic_cost_per_token(
+            model=model_name,
+            usage=combined_usage_object,
+            custom_llm_provider=custom_llm_provider,
+            data_residency=data_residency,
+        )
+    except Exception:
+        return None
+
+
+def _cost_map_entry_declares_pricing(model_name: str, custom_llm_provider: str) -> bool:
+    entries: Final = (
+        litellm.model_cost.get(model_name),
+        litellm.model_cost.get(f"{custom_llm_provider}/{model_name}"),
+    )
+    return any(
+        entry is not None and any("cost_per" in field and value is not None for field, value in entry.items())
+        for entry in entries
+    )
+
+
+def _first_priced_realtime_token_costs(
+    potential_model_names: Sequence[str | None],
+    combined_usage_object: Usage,
+    custom_llm_provider: str,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    candidate_costs: Final = (
+        (model_name, costs)
+        for model_name in potential_model_names
+        if model_name is not None
+        and (
+            costs := _candidate_realtime_token_costs(
+                model_name=model_name,
+                combined_usage_object=combined_usage_object,
+                custom_llm_provider=custom_llm_provider,
+                data_residency=data_residency,
+            )
+        )
+        is not None
+    )
+    return next(
+        (
+            costs
+            for model_name, costs in candidate_costs
+            if sum(costs) > 0 or _cost_map_entry_declares_pricing(model_name, custom_llm_provider)
+        ),
+        (0.0, 0.0),
+    )
+
+
 def handle_realtime_stream_cost_calculation(
     results: OpenAIRealtimeStreamList,
     combined_usage_object: Usage,
@@ -2360,24 +2459,12 @@ def handle_realtime_stream_cost_calculation(
             potential_model_names.append(received_model)
 
     potential_model_names.append(litellm_model_name)
-    input_cost_per_token = 0.0
-    output_cost_per_token = 0.0
-
-    for model_name in potential_model_names:
-        try:
-            if model_name is None:
-                continue
-            _input_cost_per_token, _output_cost_per_token = generic_cost_per_token(
-                model=model_name,
-                usage=combined_usage_object,
-                custom_llm_provider=custom_llm_provider,
-                data_residency=data_residency,
-            )
-        except Exception:
-            continue
-        input_cost_per_token += _input_cost_per_token
-        output_cost_per_token += _output_cost_per_token
-        break  # exit if we find a valid model
+    input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs(
+        potential_model_names=potential_model_names,
+        combined_usage_object=combined_usage_object,
+        custom_llm_provider=custom_llm_provider,
+        data_residency=data_residency,
+    )
     transcription_cost: Final = (
         handle_realtime_transcription_cost_calculation(
             results=results,

litellm/endpoints/speech/speech_to_completion_bridge/transformation.py

@@ -8,6 +8,14 @@ if TYPE_CHECKING:
     from litellm.types.utils import ModelResponse
 
 
+def _completion_response_cost(model_response: "ModelResponse") -> float | None:
+    hidden_params: Final = getattr(model_response, "_hidden_params", None)
+    if not isinstance(hidden_params, dict):
+        return None
+    response_cost: Final = hidden_params.get("response_cost")
+    return response_cost if isinstance(response_cost, float) else None
+
+
 class SpeechToCompletionBridgeTransformationHandler:
     def transform_request(
         self,
@@ -123,4 +131,6 @@ class SpeechToCompletionBridgeTransformationHandler:
 
         # Create an httpx.Response object
         response: Final = httpx.Response(status_code=200, content=binary_data, headers=headers)
-        return HttpxBinaryResponseContent(response)
+        binary_response: Final = HttpxBinaryResponseContent(response)
+        binary_response.set_response_cost(_completion_response_cost(model_response))
+        return binary_response

litellm/experimental_mcp_client/client.py

@@ -7,6 +7,7 @@ import base64
 import os
 from collections.abc import Awaitable, Callable, Generator
 from datetime import timedelta
+from importlib import metadata
 from typing import Any, Final, TypeVar
 
 import httpx
@@ -21,6 +22,18 @@ try:
     streamable_http_client = getattr(streamable_http_module, "streamable_http_client", None)
 except ImportError:
     pass
+
+MCP_STREAMABLE_HTTP_REQUIREMENT: Final = "mcp>=1.28.1"
+
+
+def missing_streamable_http_client_error() -> ImportError:
+    return ImportError(
+        f"MCP streamable HTTP transport requires {MCP_STREAMABLE_HTTP_REQUIREMENT}, but the installed "
+        f"mcp {metadata.version('mcp')} does not provide streamable_http_client. "
+        "Fix with: pip install 'litellm[mcp]' (or upgrade mcp directly: pip install -U mcp)"
+    )
+
+
 from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
 from mcp.types import CallToolResult as MCPCallToolResult
 from mcp.types import (
@@ -43,6 +56,9 @@ from litellm.types.mcp import (
     MCPStdioConfig,
     MCPTransport,
     MCPTransportType,
+    credential_redirect_hook,
+    has_header,
+    without_header,
 )
 
 
@@ -260,6 +276,7 @@ class MCPClient:
         transport_type: MCPTransportType = MCPTransport.http,
         auth_type: MCPAuthType = None,
         auth_value: str | dict[str, str] | None = None,
+        auth_header_name: str | None = None,
         timeout: float | None = None,
         stdio_config: MCPStdioConfig | None = None,
         extra_headers: dict[str, str] | None = None,
@@ -275,6 +292,11 @@ class MCPClient:
         self.auth_type: MCPAuthType = auth_type
         self.timeout: float = timeout if timeout is not None else MCP_CLIENT_TIMEOUT
         self._mcp_auth_value: str | dict[str, str] | None = None
+        # The one place this client decides which header its credential occupies: the operator's
+        # configured slot on the v1 path, or the slot the v2 resolver's auth object already owns.
+        # Every consumer reads this rather than re-deriving it, since each re-derivation so far
+        # picked up a different bug.
+        self._credential_slot: str | None = auth_header_name or getattr(resolved_auth, "header_name", None)
         self.stdio_config: MCPStdioConfig | None = stdio_config
         self.extra_headers: dict[str, str] | None = extra_headers
         self.ssl_verify: VerifyTypes | None = ssl_verify
@@ -323,7 +345,7 @@ class MCPClient:
             )
         # HTTP transport (default)
         if streamable_http_client is None:
-            raise ImportError("streamable_http_client is not available. Please install mcp with HTTP support.")
+            raise missing_streamable_http_client_error()
         headers = self._get_auth_headers()
         httpx_client_factory = self._create_httpx_client_factory()
         verbose_logger.debug("litellm headers for streamable_http_client: %s", headers)
@@ -488,26 +510,33 @@ class MCPClient:
         else:
             self._mcp_auth_value = mcp_auth_value
 
+    def _header_slot(self, default: str) -> str:
+        return self._credential_slot or default
+
     def _get_auth_headers(self) -> dict:
         """Generate authentication headers based on auth type."""
         headers: Final = {}
         if self._mcp_auth_value:
             if isinstance(self._mcp_auth_value, str):
                 if self.auth_type == MCPAuth.bearer_token:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    static_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {static_bearer}"
                 elif self.auth_type == MCPAuth.basic:
-                    headers["Authorization"] = f"Basic {self._mcp_auth_value}"
+                    headers[self._header_slot("Authorization")] = f"Basic {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.api_key:
-                    headers["X-API-Key"] = self._mcp_auth_value
+                    headers[self._header_slot("X-API-Key")] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.authorization:
                     # This auth type means the caller owns the whole header value.
-                    headers["Authorization"] = self._mcp_auth_value
+                    headers[self._header_slot("Authorization")] = self._mcp_auth_value
                 elif self.auth_type == MCPAuth.oauth2:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    oauth2_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {oauth2_bearer}"
                 elif self.auth_type == MCPAuth.token:
-                    headers["Authorization"] = f"token {strip_auth_scheme(self._mcp_auth_value, 'token')}"
+                    scheme_token: Final = strip_auth_scheme(self._mcp_auth_value, "token")
+                    headers[self._header_slot("Authorization")] = f"token {scheme_token}"
                 elif self.auth_type == MCPAuth.oauth2_token_exchange:
-                    headers["Authorization"] = f"Bearer {strip_auth_scheme(self._mcp_auth_value, 'Bearer')}"
+                    exchanged_bearer: Final = strip_auth_scheme(self._mcp_auth_value, "Bearer")
+                    headers[self._header_slot("Authorization")] = f"Bearer {exchanged_bearer}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request
@@ -515,7 +544,14 @@ class MCPClient:
         # of static headers. See MCPSigV4Auth and _create_httpx_client_factory().
         # update the headers with the extra headers
         if self.extra_headers:
-            headers.update(self.extra_headers)
+            # Mirrors _resolve_v2_auth: when the operator named a slot for the credential the
+            # gateway resolved, no injected header may shadow it, case-insensitively, since HTTP
+            # header names are. Without a configured slot the old precedence stands unchanged.
+            slot: Final = self._credential_slot
+            injected: Final = (
+                without_header(self.extra_headers, slot) if slot and has_header(headers, slot) else self.extra_headers
+            )
+            headers.update(injected or {})
         return _strip_header_whitespace(headers)
 
     def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]:
@@ -543,12 +579,14 @@ class MCPClient:
             # SigV4 aws_auth. Both are None for the common case — no behavior change.
             fallback_auth: Final = self._resolved_auth if self._resolved_auth is not None else self._aws_auth
             effective_auth: Final = auth if auth is not None else fallback_auth
+            guard: Final = credential_redirect_hook(self.server_url, self._credential_slot)
             return httpx.AsyncClient(
                 headers=headers,
                 timeout=timeout,
                 auth=effective_auth,
                 verify=ssl_config,
                 follow_redirects=True,
+                event_hooks={"request": [guard]} if guard else {},
             )
 
         return factory

litellm/google_genai/streaming_iterator.py

@@ -2,6 +2,7 @@ import asyncio
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final
 
+import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
@@ -65,6 +66,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
         litellm_logging_obj: LiteLLMLoggingObj,
         request_body: dict,
         model: str,
+        custom_llm_provider: str,
         hidden_params: dict[str, Any] | None = None,
     ):
         self.litellm_logging_obj = litellm_logging_obj
@@ -72,6 +74,10 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
         self.start_time = datetime.now()
         self.collected_chunks: list[bytes] = []
         self.model = model
+        self.custom_llm_provider = custom_llm_provider
+        self.endpoint_type: Final = (
+            EndpointType.GEMINI if custom_llm_provider == litellm.LlmProviders.GEMINI.value else EndpointType.VERTEX_AI
+        )
         self._hidden_params: dict[str, Any] = hidden_params or {}
 
     async def _handle_async_streaming_logging(
@@ -89,7 +95,7 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
                 passthrough_success_handler_obj=GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ,
                 url_route="/v1/generateContent",
                 request_body=self.request_body or {},
-                endpoint_type=EndpointType.VERTEX_AI,
+                endpoint_type=self.endpoint_type,
                 start_time=self.start_time,
                 raw_bytes=self.collected_chunks,
                 end_time=end_time,
@@ -118,13 +124,13 @@ class GoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateContent
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            custom_llm_provider=custom_llm_provider,
             hidden_params=hidden_params,
         )
         self.response = response
         self.model = model
         self.generate_content_provider_config = generate_content_provider_config
         self.litellm_metadata = litellm_metadata
-        self.custom_llm_provider = custom_llm_provider
         # Gemini streamGenerateContent uses SSE line framing; iter_lines keeps
         # large inlineData payloads (e.g. image/jpeg) intact within one event.
         self.stream_iterator = response.iter_lines()
@@ -169,13 +175,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(BaseGoogleGenAIGenerateCo
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            custom_llm_provider=custom_llm_provider,
             hidden_params=hidden_params,
         )
         self.response = response
         self.model = model
         self.generate_content_provider_config = generate_content_provider_config
         self.litellm_metadata = litellm_metadata
-        self.custom_llm_provider = custom_llm_provider
         # Gemini streamGenerateContent uses SSE line framing; aiter_lines keeps
         # large inlineData payloads (e.g. image/jpeg) intact within one event.
         self.stream_iterator = response.aiter_lines()

litellm/images/main.py

@@ -19,6 +19,7 @@ from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
 from litellm.exceptions import LiteLLMUnknownProvider
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_request_utils import flatten_form_field_values
 from litellm.litellm_core_utils.mock_functions import mock_image_generation
 from litellm.llms.base_llm import BaseImageEditConfig, BaseImageGenerationConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -422,24 +423,32 @@ def image_generation(
                 aimg_generation=aimg_generation,
             )
         elif custom_llm_provider == "azure_ai":
-            from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+            from litellm.llms.azure_ai.common_utils import (
+                AzureFoundryModelInfo,
+                get_azure_ai_auth_headers,
+            )
 
             api_base = AzureFoundryModelInfo.get_api_base(api_base)
             api_key = AzureFoundryModelInfo.get_api_key(api_key)
             if extra_headers is not None:
                 optional_params["extra_headers"] = extra_headers
 
-            default_headers = {
+            caller_header_names = frozenset(name.lower() for name in headers)
+            caller_set_auth = "api-key" in caller_header_names or "authorization" in caller_header_names
+            auth_headers = (
+                headers
+                if caller_set_auth
+                else get_azure_ai_auth_headers(
+                    api_key=api_key,
+                    litellm_params=litellm_params_dict,
+                    api_key_header="api-key",
+                )
+            )
+            request_headers: Final = {
                 "Content-Type": "application/json",
+                **auth_headers,
+                **headers,
             }
-            # Only add api-key header if api_key is not None
-            # Azure AD authentication will use Authorization header instead
-            if api_key is not None:
-                default_headers["api-key"] = api_key
-
-            for k, v in default_headers.items():
-                if k not in headers:
-                    headers[k] = v
 
             model_response = azure_chat_completions.image_generation(
                 model=model,
@@ -455,7 +464,7 @@ def image_generation(
                 api_version=api_version,
                 aimg_generation=aimg_generation,
                 client=client,
-                headers=headers,
+                headers=request_headers,
                 litellm_params=litellm_params_dict,
             )
         elif (
@@ -846,6 +855,18 @@ def image_edit(
             additional_drop_params=kwargs.get("additional_drop_params"),
         )
 
+        if (
+            custom_llm_provider == "openai"
+            or custom_llm_provider == "azure"
+            or custom_llm_provider in litellm.openai_compatible_providers
+        ):
+            image_edit_request_params.update(
+                flatten_form_field_values(
+                    non_default_params,
+                    extra_body if isinstance(extra_body, dict) else None,
+                )
+            )
+
         # Pre Call logging
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
@@ -995,6 +1016,9 @@ async def aimage_edit(
             response_format=response_format,
             size=size,
             user=user,
+            extra_headers=extra_headers,
+            extra_query=extra_query,
+            extra_body=extra_body,
             timeout=timeout,
             custom_llm_provider=custom_llm_provider,
             **kwargs,

litellm/integrations/SlackAlerting/batching_handler.py

@@ -10,6 +10,8 @@ from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_proxy_logger
 
+from .ms_teams import MS_TEAMS_ALERTING_DESTINATION, build_ms_teams_payload
+
 if TYPE_CHECKING:
     from .slack_alerting import SlackAlerting as _SlackAlerting
 
@@ -62,14 +64,17 @@ async def send_to_webhook(slackAlertingInstance: SlackAlertingType, item, count)
         if count > 1:
             payload["text"] = f"[Num Alerts: {count}]\n\n{payload['text']}"
 
+        request_body: Final = (
+            build_ms_teams_payload(payload["text"]) if item.get("format") == MS_TEAMS_ALERTING_DESTINATION else payload
+        )
         response: Final = await slackAlertingInstance.async_http_handler.post(
             url=item["url"],
             headers=item["headers"],
-            data=json.dumps(payload),
+            data=json.dumps(request_body),
         )
         if response.status_code != 200:
-            verbose_proxy_logger.debug("Error sending slack alert to url=%s. Error=%s", item["url"], response.text)
+            verbose_proxy_logger.debug("Error sending alert to url=%s. Error=%s", item["url"], response.text)
     except Exception as e:
-        verbose_proxy_logger.debug("Error sending slack alert: %s", e)
+        verbose_proxy_logger.debug("Error sending alert: %s", e)
     finally:
         _print_alerting_payload_warning(payload, slackAlertingInstance=slackAlertingInstance)

litellm/integrations/SlackAlerting/ms_teams.py

@@ -0,0 +1,75 @@
+"""Microsoft Teams alert delivery helpers.
+
+Teams incoming webhooks (Workflows and legacy connectors) accept an Adaptive
+Card wrapped in a message attachment, so alert text is delivered as a single
+wrapped TextBlock.
+"""
+
+import os
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.types.integrations.slack_alerting import AlertType
+
+MS_TEAMS_WEBHOOK_URL_ENV: Final = "MS_TEAMS_WEBHOOK_URL"
+
+MS_TEAMS_ALERTING_DESTINATION: Final = "ms_teams"
+
+MS_TEAMS_ALERT_HEADERS: Final[Mapping[str, str]] = MappingProxyType({"Content-type": "application/json"})
+
+
+class MSTeamsTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+    wrap: ReadOnly[bool]
+
+
+class MSTeamsAdaptiveCard(TypedDict):
+    type: ReadOnly[str]
+    version: ReadOnly[str]
+    body: ReadOnly[tuple[MSTeamsTextBlock, ...]]
+
+
+class MSTeamsAttachment(TypedDict):
+    contentType: ReadOnly[str]
+    content: ReadOnly[MSTeamsAdaptiveCard]
+
+
+class MSTeamsMessage(TypedDict):
+    type: ReadOnly[str]
+    attachments: ReadOnly[tuple[MSTeamsAttachment, ...]]
+
+
+class MSTeamsAlertText(TypedDict):
+    text: ReadOnly[str]
+
+
+class MSTeamsQueueItem(TypedDict):
+    url: ReadOnly[str]
+    headers: ReadOnly[Mapping[str, str]]
+    payload: ReadOnly[MSTeamsAlertText]
+    alert_type: ReadOnly[AlertType]
+    format: ReadOnly[str]
+
+
+def get_ms_teams_webhook_url() -> str | None:
+    return os.getenv(MS_TEAMS_WEBHOOK_URL_ENV)
+
+
+def build_ms_teams_payload(text: str) -> MSTeamsMessage:
+    return MSTeamsMessage(
+        type="message",
+        attachments=(
+            MSTeamsAttachment(
+                contentType="application/vnd.microsoft.card.adaptive",
+                content=MSTeamsAdaptiveCard(
+                    type="AdaptiveCard",
+                    version="1.4",
+                    body=(MSTeamsTextBlock(type="TextBlock", text=text, wrap=True),),
+                ),
+            ),
+        ),
+    )

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -57,6 +57,13 @@ from litellm.types.proxy.model_deprecation import (
 
 from ..email_templates.templates import *
 from .batching_handler import send_to_webhook, squash_payloads
+from .ms_teams import (
+    MS_TEAMS_ALERT_HEADERS,
+    MS_TEAMS_ALERTING_DESTINATION,
+    MSTeamsAlertText,
+    MSTeamsQueueItem,
+    get_ms_teams_webhook_url,
+)
 from .utils import process_slack_alerting_variables
 
 if TYPE_CHECKING:
@@ -1431,13 +1438,45 @@ Model Info:
             # only send budget alerts over Email
             await self.send_email_alert_using_smtp(webhook_event=user_info, alert_type=alert_type)
 
-        if "slack" not in self.alerting:
+        send_to_slack: Final = "slack" in self.alerting
+        send_to_ms_teams: Final = MS_TEAMS_ALERTING_DESTINATION in self.alerting
+        if not send_to_slack and not send_to_ms_teams:
             return
         if alert_type not in self.alert_types:
             return
 
         from datetime import datetime
 
+        # Get the current timestamp
+        current_time: Final = datetime.now().strftime("%H:%M:%S")
+        _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
+        # Use .name if it's an enum, otherwise use as is
+        alert_type_name: Final = getattr(alert_type, "name", alert_type)
+        alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
+        if alert_type == "daily_reports" or alert_type == "new_model_added":
+            formatted_message = alert_type_formatted + message
+        else:
+            formatted_message = (
+                f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
+            )
+
+        if kwargs:
+            for key, value in kwargs.items():
+                formatted_message += f"\n\n{key}: `{value}`\n\n"
+        if alerting_metadata:
+            for key, value in alerting_metadata.items():
+                formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
+        if _proxy_base_url is not None:
+            formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
+
+        if send_to_ms_teams:
+            self._enqueue_ms_teams_alert(formatted_message=formatted_message, alert_type=alert_type)
+
+        if not send_to_slack:
+            if len(self.log_queue) >= self.batch_size:
+                await self.flush_queue()
+            return
+
         # Check if digest mode is enabled for this alert type
         alert_type_name_str: Final = getattr(alert_type, "value", str(alert_type))
         _atc: Final = self.alert_type_config.get(alert_type_name_str)
@@ -1473,28 +1512,6 @@ Model Info:
                     )
             return  # Suppress immediate alert; will be emitted by _flush_digest_buckets
 
-        # Get the current timestamp
-        current_time: Final = datetime.now().strftime("%H:%M:%S")
-        _proxy_base_url: Final = os.getenv("PROXY_BASE_URL", None)
-        # Use .name if it's an enum, otherwise use as is
-        alert_type_name: Final = getattr(alert_type, "name", alert_type)
-        alert_type_formatted: Final = f"Alert type: `{alert_type_name}`"
-        if alert_type == "daily_reports" or alert_type == "new_model_added":
-            formatted_message = alert_type_formatted + message
-        else:
-            formatted_message = (
-                f"{alert_type_formatted}\nLevel: `{level}`\nTimestamp: `{current_time}`\n\nMessage: {message}"
-            )
-
-        if kwargs:
-            for key, value in kwargs.items():
-                formatted_message += f"\n\n{key}: `{value}`\n\n"
-        if alerting_metadata:
-            for key, value in alerting_metadata.items():
-                formatted_message += f"\n\n*Alerting Metadata*: \n{key}: `{value}`\n\n"
-        if _proxy_base_url is not None:
-            formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
-
         # check if we find the slack webhook url in self.alert_to_webhook_url
         if self.alert_to_webhook_url is not None and alert_type in self.alert_to_webhook_url:
             slack_webhook_url: str | list[str] | None = self.alert_to_webhook_url[alert_type]
@@ -1531,6 +1548,24 @@ Model Info:
         if len(self.log_queue) >= self.batch_size:
             await self.flush_queue()
 
+    def _enqueue_ms_teams_alert(self, formatted_message: str, alert_type: AlertType) -> None:
+        ms_teams_webhook_url: Final = get_ms_teams_webhook_url()
+        if ms_teams_webhook_url is None:
+            verbose_proxy_logger.error(
+                "MS Teams alerting is enabled but MS_TEAMS_WEBHOOK_URL is not set. Dropping alert type=%s",
+                alert_type,
+            )
+            return
+        payload: Final[MSTeamsAlertText] = {"text": formatted_message}
+        item: Final[MSTeamsQueueItem] = {
+            "url": ms_teams_webhook_url,
+            "headers": MS_TEAMS_ALERT_HEADERS,
+            "payload": payload,
+            "alert_type": alert_type,
+            "format": MS_TEAMS_ALERTING_DESTINATION,
+        }
+        self.log_queue.append(item)
+
     async def async_send_batch(self):
         if not self.log_queue:
             return

litellm/integrations/anthropic_cache_control_hook.py

@@ -24,6 +24,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     with_prompt_cache_breakpoint,
 )
 from litellm.types.integrations.anthropic_cache_control_hook import (
+    GATEWAY_INJECTED_CACHE_METADATA_KEY,
+    GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT,
     CacheControlInjectionPoint,
     CacheControlMessageInjectionPoint,
 )
@@ -104,6 +106,13 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
     return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
 
 
+# Set by a caller whose message list is not the one that goes upstream -- today the
+# Responses API layer, whose `instructions` only becomes a system message further down.
+# Tells this hook to hand role-targeted points to the pass holding the final messages
+# rather than spending them on a list that is still missing some of their targets.
+CARRY_UNMATCHED_MESSAGE_POINTS: Final = "_litellm_carry_unmatched_cache_control_points"
+
+
 class AnthropicCacheControlHook(CustomPromptManagement):
     def get_chat_completion_prompt(
         self,
@@ -128,6 +137,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         - non_default_params: dict - params with any global cache controls
         """
         # Extract cache control injection points
+        carry_unmatched: Final = bool(non_default_params.pop(CARRY_UNMATCHED_MESSAGE_POINTS, False))
         injection_points: Final[list[CacheControlInjectionPoint]] = non_default_params.pop(
             "cache_control_injection_points", []
         )
@@ -161,26 +171,44 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 non_default_params.get("prompt_cache_options"),
             )
         )
+        # A provisional message list defers every role-targeted point to the pass holding
+        # the final one: a role with no message here may have one there, and settling all
+        # of them in one pass is what lets config order decide the shared breakpoint
+        # budget. An ordinal names a different message once a later layer builds its own
+        # list, so it is placed here or not at all.
+        carried_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
+            tuple(point for point in message_points if point.get("index") is None) if carry_unmatched else ()
+        )
+        applied_message_points: Final[Sequence[CacheControlMessageInjectionPoint]] = (
+            tuple(point for point in message_points if point.get("index") is not None)
+            if carry_unmatched
+            else tuple(message_points)
+        )
         reserved_blocks: Final = (
             1 if not openai_dialect and any(p.get("location") == "tool_config" for p in remaining_points) else 0
         )
-        breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
+        breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
         processed_messages = self._apply_message_injections(
-            points=message_points,
+            points=applied_message_points,
             messages=processed_messages,
             max_blocks=MAX_CACHE_CONTROL_BLOCKS - reserved_blocks,
             openai_dialect=openai_dialect,
         )
         if (
             openai_dialect
-            and AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages) > breakpoints_before
+            and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before
         ):
             non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
 
-        # Pass through non-message injection points for provider-specific handling
-        if remaining_points:
+        # Points this pass did not place: non-message ones for the provider transform, and
+        # the deferred role-targeted ones. Deferring is what reaches the Responses API's
+        # `instructions`, which is only a system message once the bridge builds one. The
+        # judged stamp is what makes it safe: the next pass must not re-judge points
+        # against messages this pass already marked (see `_should_stand_down`).
+        carried_points: Final[Sequence[CacheControlInjectionPoint]] = (*remaining_points, *carried_message_points)
+        if carried_points:
             non_default_params["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(
-                remaining_points
+                carried_points
             )
 
         return model, processed_messages, non_default_params
@@ -210,7 +238,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         return provider
 
     @staticmethod
-    def _count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
+    def count_request_cache_breakpoints(messages: Iterable[object], system: object = None) -> int:
         system_blocks: Final = (
             sum(1 for block in system if _carries_cache_breakpoint(block)) if isinstance(system, list) else 0
         )
@@ -218,7 +246,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
 
     @staticmethod
     def _apply_message_injections(
-        points: list[CacheControlMessageInjectionPoint],
+        points: Sequence[CacheControlMessageInjectionPoint],
         messages: list[AllMessageValues],
         max_blocks: int,
         openai_dialect: bool = False,
@@ -232,7 +260,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         ``max_blocks`` is reached. Injection points are honored in config order,
         so earlier points win when slots are scarce.
         """
-        used_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints(messages)
+        used_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints(messages)
 
         limit_reached = False
         for point in points:
@@ -350,7 +378,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         # 2. list of objects - only apply to last item per Anthropic spec
         elif isinstance(message_content, list):
             if len(message_content) > 0 and isinstance(message_content[-1], dict):
-                message_content[-1]["cache_control"] = control
+                message_content[-1]["cache_control"] = control  # pyright: ignore[reportGeneralTypeIssues]  # loose runtime dict
         return message
 
     @staticmethod
@@ -428,8 +456,8 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         max_blocks: Final = MAX_CACHE_CONTROL_BLOCKS - reserved_blocks
 
-        message_blocks: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(processed_messages)
-        system_blocks = AnthropicCacheControlHook._count_request_cache_breakpoints((), processed_system)
+        message_blocks: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages)
+        system_blocks = AnthropicCacheControlHook.count_request_cache_breakpoints((), processed_system)
 
         if system_points and processed_system is not None and message_blocks + system_blocks < max_blocks:
             system_already_has_cc: Final = isinstance(processed_system, list) and any(
@@ -563,7 +591,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         carry the mark either at the top level (Anthropic shape) or nested under
         ``function`` (OpenAI shape); the Anthropic chat transform accepts both.
         """
-        if AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > 0:
+        if AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) > 0:
             return True
         if tools is not None:
             return any(
@@ -723,6 +751,64 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         if points:
             non_default_params["cache_control_injection_points"] = points
 
+    @staticmethod
+    def record_gateway_injection(
+        request_kwargs: Mapping[str, object],
+        added: int,
+    ) -> None:
+        """Name the deployment whose payload the gateway, not the client, put breakpoints on.
+
+        Spend accounting only asks whether litellm acted, so what it needs is which
+        deployment, not a count. Recording that is what makes the mark attempt-scoped: the
+        metadata bucket is one dict shared by every retry, failover and fallback of a
+        request, and ``litellm_call_id`` is shared with it, so anything request-scoped
+        written by one attempt is read by all of them and each boundary would have to
+        remember to strip it. The deployment is the part that actually changes when the
+        request moves, so a leg that injected nothing is never credited for one that did.
+
+        It also makes a zero delta (hook re-entry) and a negative one (a prompt manager
+        replacing the messages) harmless, since neither rewrites an earlier mark.
+
+        A pass that runs before a deployment is chosen, which is what the proxy does for
+        prompt templates, injects into the payload every leg goes on to send, so it marks
+        the request for all of them rather than for one.
+
+        Only what this pass actually placed counts. A ``tool_config`` point is placed by
+        the Bedrock converse transform, and only when the request carries tools, so the
+        presence of one here says nothing about whether a breakpoint reaches the wire;
+        claiming it marked three request shapes out of four that inject nothing. Missing
+        that Bedrock credit is the fail-closed direction, and the alternative is a
+        provider transform that carries spend-attribution state.
+
+        Reads whichever bucket the request actually carries rather than asking the shared
+        name resolver, which answers on key presence: ``litellm_params`` declares
+        ``litellm_metadata`` as None on every request, so the resolver names a bucket that
+        is not there and the mark is dropped.
+
+        Never CREATES the bucket. The proxy seeds it on every request and is the marker's
+        only reader, so a request without one is a bare SDK call nothing would consume it
+        from. Creating it would also add a key to a dict call sites splat as ``**kwargs``,
+        and on the Responses API ``metadata`` is both this bucket's default name and an
+        explicit parameter, so the splat collides with the caller's own value.
+        """
+        if added <= 0:
+            return
+        bucket: Final = next(
+            (
+                candidate
+                for candidate in (request_kwargs.get("litellm_metadata"), request_kwargs.get("metadata"))
+                if isinstance(candidate, dict)
+            ),
+            None,
+        )
+        if bucket is not None:
+            model_info: Final = request_kwargs.get("model_info")
+            bucket[GATEWAY_INJECTED_CACHE_METADATA_KEY] = (
+                model_info.get("id", GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT)
+                if isinstance(model_info, dict)
+                else GATEWAY_INJECTED_FOR_EVERY_DEPLOYMENT
+            )
+
     @staticmethod
     def maybe_inject_cache_control(
         messages: list[dict],
@@ -772,17 +858,18 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         openai_dialect: Final = AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(
             model, custom_llm_provider, api_base, kwargs.get("prompt_cache_options")
         )
-        breakpoints_before: Final = AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system)
+        breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system)
         messages, system, remaining = AnthropicCacheControlHook.apply_to_anthropic_messages_request(
             messages=messages,
             system=system,
             injection_points=injection_points,
             openai_dialect=openai_dialect,
         )
-        if (
-            openai_dialect
-            and AnthropicCacheControlHook._count_request_cache_breakpoints(messages, system) > breakpoints_before
-        ):
+        breakpoints_added: Final = (
+            AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before
+        )
+        AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added)
+        if openai_dialect and breakpoints_added > 0:
             kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="explicit"))
         if remaining:
             kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_as_judged(remaining)

litellm/integrations/callback_configs.json

@@ -220,6 +220,12 @@
         "ui_name": "Host URL",
         "description": "Langfuse host URL (default: https://cloud.langfuse.com)",
         "required": false
+      },
+      "langfuse_environment": {
+        "type": "text",
+        "ui_name": "Tracing Environment",
+        "description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
+        "required": false
       }
     },
     "description": "Langfuse v2 Logging Integration"
@@ -247,6 +253,12 @@
         "ui_name": "Host URL",
         "description": "Langfuse host URL (default: https://cloud.langfuse.com)",
         "required": false
+      },
+      "langfuse_environment": {
+        "type": "text",
+        "ui_name": "Tracing Environment",
+        "description": "Langfuse tracing environment (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)",
+        "required": false
       }
     },
     "description": "Langfuse v3 OTEL Logging Integration"
@@ -294,12 +306,18 @@
     "id": "newrelic",
     "displayName": "New Relic",
     "logo": "newrelic.png",
-    "supports_key_team_logging": false,
+    "supports_key_team_logging": true,
     "dynamic_params": {
-      "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
+      "newrelic_api_key": {
+        "type": "password",
+        "ui_name": "New Relic Ingest License Key",
+        "description": "Per-team ingest (license) key. Team traces export to this key's New Relic account over OTLP.",
+        "required": false
+      },
+      "newrelic_region": {
         "type": "text",
-        "ui_name": "Record AI Content (default: true)",
-        "description": "Whether to record AI message content. Set to false to disable.",
+        "ui_name": "New Relic Region (us or eu)",
+        "description": "Data center region for this team's account. Defaults to us.",
         "required": false
       }
     },

litellm/integrations/compression_interception/handler.py

@@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
 
 import time
 import uuid
-from typing import Any, Final, cast
+from typing import Any, ClassVar, Final, cast
 
 from litellm._logging import verbose_logger
 from litellm.compression import compress
@@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
     4. Build typed rerun plan with tool_result blocks from the compressed cache.
     """
 
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
+
     def __init__(
         self,
         enabled: bool = True,

litellm/integrations/custom_batch_logger.py

@@ -45,7 +45,7 @@ class CustomBatchLogger(CustomLogger):
 
         super().__init__(**kwargs)
 
-    async def periodic_flush(self):
+    async def periodic_flush(self) -> None:
         while True:
             await asyncio.sleep(self.flush_interval)
             verbose_logger.debug("CustomLogger periodic flush after %s seconds", self.flush_interval)

litellm/integrations/custom_logger.py

@@ -2,8 +2,8 @@
 #    On success, logs events to Promptlayer
 import re
 import traceback
-from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from collections.abc import AsyncGenerator, Mapping
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
 
@@ -60,6 +60,7 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
 
     enforces_request_content: bool = False
     """
@@ -292,6 +293,54 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
 
+    async def async_post_call_failure_deployment_hook(
+        self,
+        request_data: Mapping[str, object],
+        exception: Exception,
+        call_type: CallTypes | None,
+        fallback_depth: int | None = None,
+    ) -> None:
+        """
+        Called once per failed deployment attempt - attempt 1, every retry, and
+        every fallback chain step - because the router re-invokes the wrapped
+        function on each attempt, re-entering this hook's call site fresh
+        every time.
+
+        This is a DEPLOYMENT-LEVEL signal, distinct from the REQUEST-LEVEL
+        ``async_log_failure_event``, which fires once per logical client
+        request behind a dedup gate. ``request_data`` is mostly this
+        attempt's own kwargs, with one exception: it omits
+        ``attempted_targets``, the router's own bookkeeping of which fallback
+        targets this request has already tried, since that one object *is*
+        shared by reference across every hop of the live fallback walk.
+
+        Pairs with ``async_pre_call_deployment_hook`` and
+        ``async_post_call_success_deployment_hook`` to complete the
+        pre-call/success/failure lifecycle for a single deployment attempt.
+
+        ``fallback_depth`` is best-effort: ``None`` on the first attempt and on
+        any call made without a ``Router`` (a bare SDK call has no fallback
+        chain to be at a depth in), ``1`` on the first fallback hop, ``2`` on
+        the second, and so on. It reflects ``Router``'s own internal fallback
+        bookkeeping (``kwargs["fallback_depth"]``), not a value this hook
+        computes or guarantees the shape of across versions. It tracks
+        fallback hops only, not retries within the same model group - a
+        retry-only failure (no fallback yet) also reports ``None``. If an
+        override predates this field it's simply never passed, rather than
+        raising - safe to leave off an override written before it existed.
+
+        ``exception`` is a same-class snapshot, not the exact object about to
+        be re-raised to the real caller: read it freely, but setting an
+        attribute on it (e.g. ``status_code``) has no effect on what the
+        caller actually receives.
+
+        Default: no-op. Opt in by overriding. Keep overrides fast - this
+        runs on the request's exception path, so a slow implementation
+        delays error propagation to the caller. The reported failure
+        duration is captured before this hook runs, so a slow override
+        doesn't inflate that metric, but the caller still waits for it.
+        """
+
     async def async_post_call_streaming_deployment_hook(
         self,
         request_data: dict,

litellm/integrations/dotprompt/__init__.py

@@ -62,12 +62,16 @@ def prompt_initializer(litellm_params: "PromptLiteLLMParams", prompt_spec: "Prom
     if dotprompt_content and not prompt_data and not prompt_file:
         prompt_data = _get_prompt_data_from_dotprompt_content(dotprompt_content)
 
+    from .prompt_manager import strip_version_suffix
+
+    registration_prompt_id: Final = prompt_id or strip_version_suffix(prompt_spec.prompt_id) or prompt_spec.prompt_id
+
     try:
         dot_prompt_manager: Final = DotpromptManager(
             prompt_directory=prompt_directory,
             prompt_data=prompt_data,
             prompt_file=prompt_file,
-            prompt_id=prompt_id,
+            prompt_id=registration_prompt_id,
         )
 
         return dot_prompt_manager

litellm/integrations/dotprompt/dotprompt_manager.py

@@ -96,7 +96,7 @@ class DotpromptManager(CustomPromptManagement):
         if prompt_id is None:
             return False
         try:
-            return prompt_id in self.prompt_manager.list_prompts()
+            return self.prompt_manager.get_prompt(prompt_id) is not None
         except Exception:
             # If there's any error accessing prompts, don't run prompt management
             return False
@@ -209,6 +209,8 @@ class DotpromptManager(CustomPromptManagement):
             prompt_spec=prompt_spec,
             prompt_label=prompt_label,
             prompt_version=prompt_version,
+            ignore_prompt_manager_model=ignore_prompt_manager_model,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
         )
 
     async def async_get_chat_completion_prompt(

litellm/integrations/dotprompt/prompt_manager.py

@@ -11,6 +11,13 @@ from jinja2 import DictLoader, select_autoescape
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 
+def strip_version_suffix(prompt_id: str) -> str | None:
+    base, separator, version = prompt_id.rpartition(".v")
+    if separator and base and version.isdigit():
+        return base
+    return None
+
+
 class PromptTemplate:
     """Represents a single prompt template with metadata and content."""
 
@@ -124,11 +131,13 @@ class PromptManager:
             "content": "template content",
             "metadata": {"model": "gpt-4", "temperature": 0.7, ...}
         } + prompt_id
-        """
-        if prompt_id:
-            prompt_data = {prompt_id: prompt_data}
 
-        for prompt_id, prompt_info in prompt_data.items():
+        A dict carrying a "content" key is a single flat template registered under
+        prompt_id; anything else is treated as already keyed by template ID.
+        """
+        keyed_prompts: Final = {prompt_id: prompt_data} if prompt_id and "content" in prompt_data else prompt_data
+
+        for template_id, prompt_info in keyed_prompts.items():
             try:
                 content = prompt_info.get("content", "")
                 metadata = prompt_info.get("metadata", {})
@@ -136,11 +145,10 @@ class PromptManager:
                 template = PromptTemplate(
                     content=content,
                     metadata=metadata,
-                    template_id=prompt_id,
+                    template_id=template_id,
                 )
-                self.prompts[prompt_id] = template
+                self.prompts[template_id] = template
             except Exception:
-                # Optional: print(f"Error loading prompt from JSON: {prompt_id}")
                 pass
 
     def _load_prompt_file(self, file_path: str | Path, prompt_id: str) -> PromptTemplate:
@@ -272,8 +280,12 @@ class PromptManager:
             if versioned_id in self.prompts:
                 return self.prompts[versioned_id]
 
-        # Fall back to base prompt_id
-        return self.prompts.get(prompt_id)
+        direct_match: Final = self.prompts.get(prompt_id)
+        if direct_match is not None:
+            return direct_match
+
+        base_prompt_id: Final = strip_version_suffix(prompt_id)
+        return self.prompts.get(base_prompt_id) if base_prompt_id else None
 
     def list_prompts(self) -> list[str]:
         """Get a list of all available prompt IDs."""

litellm/integrations/generic_prompt_management/generic_prompt_manager.py

@@ -416,17 +416,8 @@ class GenericPromptManager(CustomPromptManagement):
             tools=tools,
             prompt_label=prompt_label,
             prompt_version=prompt_version,
-            ignore_prompt_manager_model=(
-                ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
-                if prompt_spec
-                else False
-            ),
-            ignore_prompt_manager_optional_params=(
-                ignore_prompt_manager_optional_params
-                or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
-                if prompt_spec
-                else False
-            ),
+            ignore_prompt_manager_model=ignore_prompt_manager_model,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
         )
 
     def get_chat_completion_prompt(
@@ -457,17 +448,8 @@ class GenericPromptManager(CustomPromptManagement):
             prompt_spec=prompt_spec,
             prompt_label=prompt_label,
             prompt_version=prompt_version,
-            ignore_prompt_manager_model=(
-                ignore_prompt_manager_model or prompt_spec.litellm_params.ignore_prompt_manager_model
-                if prompt_spec
-                else False
-            ),
-            ignore_prompt_manager_optional_params=(
-                ignore_prompt_manager_optional_params
-                or prompt_spec.litellm_params.ignore_prompt_manager_optional_params
-                if prompt_spec
-                else False
-            ),
+            ignore_prompt_manager_model=ignore_prompt_manager_model,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
         )
 
     def clear_cache(self) -> None:

litellm/integrations/langfuse/langfuse.py

@@ -1,9 +1,11 @@
 #### What this does ####
 #    On success, logs events to Langfuse
+import inspect
 import os
 import traceback
 from collections.abc import Callable, Iterable, Mapping
 from datetime import datetime
+from functools import lru_cache
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast
 
@@ -21,6 +23,9 @@ from litellm.litellm_core_utils.core_helpers import (
     reconstruct_model_name,
     safe_deep_copy,
 )
+from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+    validate_langfuse_environment_value,
+)
 from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info
 from litellm.llms.custom_httpx.http_handler import _get_httpx_client
 from litellm.secret_managers.main import str_to_bool
@@ -133,6 +138,16 @@ def resolve_langfuse_credentials(
     return public_key, secret_key, resolved_host
 
 
+@lru_cache(maxsize=8)
+def _warn_invalid_deployment_environment(raw_value: str, error: str) -> None:
+    verbose_logger.warning(
+        "Ignoring invalid LANGFUSE_TRACING_ENVIRONMENT=%r for the langfuse callback: %s. "
+        "Traces will be sent to Langfuse's default environment.",
+        raw_value,
+        error,
+    )
+
+
 class LangFuseLogger:
     # Class variables or attributes
     def __init__(
@@ -140,6 +155,7 @@ class LangFuseLogger:
         langfuse_public_key=None,
         langfuse_secret=None,
         langfuse_host=None,
+        langfuse_environment: str | None = None,
         flush_interval=1,
         allow_env_credentials: bool = True,
     ):
@@ -159,6 +175,12 @@ class LangFuseLogger:
         if not (self.langfuse_host.startswith("http://") or self.langfuse_host.startswith("https://")):
             # add http:// if unset, assume communicating over private network - e.g. render
             self.langfuse_host = "http://" + self.langfuse_host
+        _env_override: Final = str(langfuse_environment).strip() if langfuse_environment is not None else None
+        if _env_override:
+            validate_langfuse_environment_value(_env_override)
+            self.langfuse_environment: str | None = _env_override
+        else:
+            self.langfuse_environment = self.resolve_deployment_environment()
         self.langfuse_release = os.getenv("LANGFUSE_RELEASE")
         self.langfuse_debug = os.getenv("LANGFUSE_DEBUG")
         self.langfuse_flush_interval = LangFuseLogger._get_langfuse_flush_interval(flush_interval)
@@ -182,6 +204,8 @@ class LangFuseLogger:
         }
         self.langfuse_sdk_version: str = langfuse.version.__version__
 
+        if "environment" in inspect.signature(Langfuse.__init__).parameters:
+            parameters["environment"] = self.langfuse_environment
         if Version(self.langfuse_sdk_version) >= Version("2.6.0"):
             parameters["sdk_integration"] = "litellm"
         self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters)
@@ -942,6 +966,20 @@ class LangFuseLogger:
             verbose_logger.warning("Failed to apply masking function: %s. Returning original data.", e)
             return data
 
+    @staticmethod
+    def resolve_deployment_environment() -> str | None:
+        """Resolve LANGFUSE_TRACING_ENVIRONMENT: stripped value, "default" plus a warning when invalid, None when unset."""
+        raw: Final = os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
+        if not raw:
+            return None
+        value: Final = raw.strip()
+        try:
+            validate_langfuse_environment_value(value)
+        except ValueError as e:
+            _warn_invalid_deployment_environment(raw, str(e))
+            return "default"
+        return value
+
     @staticmethod
     def _get_langfuse_flush_interval(flush_interval: int) -> int:
         """

litellm/integrations/langfuse/langfuse_handler.py

@@ -6,6 +6,7 @@ Used to get the LangFuseLogger for a given request
 Handles Key/Team Based Langfuse Logging
 """
 
+import os
 from typing import TYPE_CHECKING, Any, Final
 
 from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
@@ -108,6 +109,7 @@ class LangFuseHandler:
             langfuse_public_key=credentials.get("langfuse_public_key"),
             langfuse_secret=credentials.get("langfuse_secret") or credentials.get("langfuse_secret_key"),
             langfuse_host=credentials.get("langfuse_host"),
+            langfuse_environment=credentials.get("langfuse_environment"),
             allow_env_credentials=credentials.get("langfuse_host") is None,
         )
         in_memory_dynamic_logger_cache.set_cache(
@@ -135,8 +137,33 @@ class LangFuseHandler:
             or standard_callback_dynamic_params.get("langfuse_secret_key"),
             langfuse_public_key=standard_callback_dynamic_params.get("langfuse_public_key"),
             langfuse_host=standard_callback_dynamic_params.get("langfuse_host"),
+            langfuse_environment=LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params),
         )
 
+    @staticmethod
+    def _meaningful_dynamic_environment(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+    ) -> str | None:
+        """Return the per-request environment only when it changes behavior.
+
+        Empty/whitespace values and values equal to the deployment-wide
+        LANGFUSE_TRACING_ENVIRONMENT fallback are treated as absent so an
+        environment-only override that matches the default does not mint a
+        duplicate SDK client (each client costs threads and counts against
+        MAX_LANGFUSE_INITIALIZED_CLIENTS).
+        """
+        raw = standard_callback_dynamic_params.get("langfuse_environment")
+        if raw is None:
+            return None
+        value = str(raw).strip()
+        if (
+            not value
+            or value == os.getenv("LANGFUSE_TRACING_ENVIRONMENT")
+            or value == LangFuseLogger.resolve_deployment_environment()
+        ):
+            return None
+        return value
+
     @staticmethod
     def _dynamic_langfuse_credentials_are_passed(
         standard_callback_dynamic_params: StandardCallbackDynamicParams,
@@ -153,6 +180,7 @@ class LangFuseHandler:
             or standard_callback_dynamic_params.get("langfuse_public_key") is not None
             or standard_callback_dynamic_params.get("langfuse_secret") is not None
             or standard_callback_dynamic_params.get("langfuse_secret_key") is not None
+            or LangFuseHandler._meaningful_dynamic_environment(standard_callback_dynamic_params) is not None
         ):
             return True
         return False

litellm/integrations/langfuse/langfuse_otel.py

@@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
     LangfuseLLMObsOTELAttributes,
 )
 from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.types.integrations.langfuse_otel import (
     LangfuseSpanAttributes,
 )
@@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            safe_json_loads(arguments_str, default={})
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),
@@ -226,7 +231,10 @@ class LangfuseOtelLogger(OpenTelemetry):
         from litellm.integrations.arize._utils import safe_set_attribute
         from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
-        langfuse_environment: Final = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
+        dynamic_params: Final = kwargs.get("standard_callback_dynamic_params")
+        langfuse_environment: Final = (
+            dynamic_params.get("langfuse_environment") if dynamic_params else None
+        ) or os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
         if langfuse_environment:
             safe_set_attribute(
                 span,

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -2,6 +2,7 @@
 Call Hook for LiteLLM Proxy which allows Langfuse prompt management.
 """
 
+import inspect
 import os
 from functools import lru_cache
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
@@ -109,6 +110,9 @@ def langfuse_client_init(
             cert=os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate),
         )
 
+    if "environment" in inspect.signature(Langfuse.__init__).parameters:
+        parameters["environment"] = LangFuseLogger.resolve_deployment_environment()
+
     client: Final = Langfuse(**parameters)
 
     return client

litellm/integrations/langsmith.py

@@ -168,17 +168,20 @@ class LangsmithLogger(CustomBatchLogger):
         return outputs
 
     def _ensure_required_ids(self, data: dict, run_id: str | None):
+        resolved_id: Final = run_id or str(uuid.uuid4())
         if "id" not in data or data["id"] is None:
-            run_id = str(uuid.uuid4())
-            data["id"] = run_id
+            data["id"] = resolved_id
 
-        if "trace_id" not in data or data["trace_id"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["trace_id"] = run_id
+        # LangSmith rejects the whole ingest batch unless a root run's trace_id
+        # equals the run id embedded in the first segment of dotted_order
+        posts_as_root: Final = ("parent_run_id" not in data or data["parent_run_id"] is None) and (
+            "dotted_order" not in data or data["dotted_order"] is None
+        )
+        if posts_as_root or "trace_id" not in data or data["trace_id"] is None:
+            data["trace_id"] = resolved_id
 
         if "dotted_order" not in data or data["dotted_order"] is None:
-            if run_id is not None and isinstance(run_id, str):
-                data["dotted_order"] = self.make_dot_order(run_id=run_id)
+            data["dotted_order"] = self.make_dot_order(run_id=resolved_id)
 
     def _prepare_log_data(
         self,
@@ -193,6 +196,11 @@ class LangsmithLogger(CustomBatchLogger):
             metadata = _litellm_params.get("metadata", {}) or {}
 
             fields: Final = self._extract_metadata_fields(metadata, credentials)
+            # the proxy header fan-out mirrors one value into both keys, and LangSmith
+            # rejects the whole ingest batch when run-body session_id is not an
+            # existing tracer-session uuid
+            if fields["session_id"] == fields["trace_id"]:
+                fields["session_id"] = None
             verbose_logger.debug(
                 "Langsmith Logging - project_name: %s, run_name %s", fields["project_name"], fields["run_name"]
             )

litellm/integrations/newrelic/newrelic_metrics.py

@@ -0,0 +1,395 @@
+"""
+New Relic Metric API Integration - sends per-team cost/usage metrics to /metric/v1
+
+NR Reference API: https://docs.newrelic.com/docs/data-apis/ingest-apis/metric-api/introduction-metric-api/
+
+`async_log_success_event` / `async_log_failure_event` queue one record per request;
+at flush the queue is aggregated by (team, model group, model, provider, status)
+into count/summary metrics. `interval.ms` is the real window between flushes,
+computed at flush time.
+
+Team-scoped by construction: the ingest key is injected explicitly and there is
+deliberately no environment-variable fallback, so a team's metrics are never sent
+with the proxy operator's credentials (mirrors ``allow_env_credentials=False`` on
+the Datadog team logger).
+
+Error policy on flush: 4xx drops the batch (a retry would fail identically; 403
+is a permanent credential failure), 5xx/network re-queues capped at
+``max_queue_size`` records with the oldest dropped.
+
+For batching specific details see CustomBatchLogger class
+"""
+
+import asyncio
+import gzip
+import time
+import traceback
+from collections.abc import Mapping
+from math import ceil
+from types import MappingProxyType
+from typing import Final
+
+from httpx import HTTPStatusError, Response
+
+from litellm._logging import verbose_logger
+from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,
+    httpxSpecialProvider,
+)
+from litellm.types.integrations.newrelic import (
+    NEWRELIC_DEFAULT_REGION,
+    NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN,
+    NEWRELIC_METRIC_COMPLETION_TOKENS,
+    NEWRELIC_METRIC_COST_USD,
+    NEWRELIC_METRIC_ENDPOINT_BY_REGION,
+    NEWRELIC_METRIC_PROMPT_TOKENS,
+    NEWRELIC_METRIC_REQUEST_DURATION_MS,
+    NEWRELIC_METRIC_REQUESTS,
+    NEWRELIC_METRIC_TOTAL_TOKENS,
+    NEWRELIC_METRICS_MAX_BATCH_SIZE,
+    NEWRELIC_METRICS_MAX_DRAIN_PASSES,
+    NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
+    NewRelicCountMetric,
+    NewRelicMetric,
+    NewRelicMetricCommon,
+    NewRelicMetricEnvelope,
+    NewRelicMetricRecord,
+    NewRelicSummaryMetric,
+    NewRelicSummaryValue,
+)
+from litellm.types.utils import StandardLoggingPayload
+
+# 408 (request timeout) and 429 (rate limit) are transient client errors the
+# Metric API expects a retry on, unlike 400/403 which a retry would only repeat.
+_RETRYABLE_CLIENT_STATUSES: Final = frozenset({408, 429})
+
+
+def resolve_newrelic_metric_endpoint(newrelic_region: str | None) -> str:
+    if not newrelic_region:
+        return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
+    endpoint: Final = NEWRELIC_METRIC_ENDPOINT_BY_REGION.get(newrelic_region.lower())
+    if endpoint is None:
+        verbose_logger.warning(
+            "New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
+            newrelic_region,
+            ", ".join(sorted(NEWRELIC_METRIC_ENDPOINT_BY_REGION)),
+        )
+        return NEWRELIC_METRIC_ENDPOINT_BY_REGION[NEWRELIC_DEFAULT_REGION]
+    return endpoint
+
+
+def _metric_record_from_payload(standard_logging_object: StandardLoggingPayload) -> NewRelicMetricRecord:
+    metadata: Final = standard_logging_object.get("metadata")
+    team_id: Final = ((metadata.get("user_api_key_team_id") or metadata.get("team_id")) if metadata else None) or ""
+    team_alias: Final = (
+        (metadata.get("user_api_key_team_alias") or metadata.get("team_alias")) if metadata else None
+    ) or ""
+    return NewRelicMetricRecord(
+        team_id=team_id,
+        team_alias=team_alias,
+        model_group=standard_logging_object.get("model_group") or "",
+        model=standard_logging_object.get("model") or "",
+        custom_llm_provider=standard_logging_object.get("custom_llm_provider") or "",
+        status=str(standard_logging_object.get("status") or "success"),
+        response_cost=float(standard_logging_object.get("response_cost") or 0.0),
+        prompt_tokens=int(standard_logging_object.get("prompt_tokens") or 0),
+        completion_tokens=int(standard_logging_object.get("completion_tokens") or 0),
+        total_tokens=int(standard_logging_object.get("total_tokens") or 0),
+        duration_ms=float(standard_logging_object.get("response_time") or 0.0) * 1000.0,
+    )
+
+
+def _bucket_metrics(bucket_records: tuple[NewRelicMetricRecord, ...]) -> tuple[NewRelicMetric, ...]:
+    first: Final = bucket_records[0]
+    attributes: Final[Mapping[str, str]] = {  # mutable-ok: JSON leaf; safe_dumps stringifies MappingProxyType
+        key: value[:NEWRELIC_METRIC_ATTRIBUTE_MAX_LEN]
+        for key, value in (
+            ("team_id", first.team_id),
+            ("team_alias", first.team_alias),
+            ("model_group", first.model_group),
+            ("model", first.model),
+            ("custom_llm_provider", first.custom_llm_provider),
+            ("status", first.status),
+        )
+        if value
+    }
+    durations: Final = tuple(record.duration_ms for record in bucket_records)
+    counts: Final[tuple[tuple[str, float], ...]] = (
+        (NEWRELIC_METRIC_REQUESTS, float(len(bucket_records))),
+        (NEWRELIC_METRIC_COST_USD, sum(record.response_cost for record in bucket_records)),
+        (NEWRELIC_METRIC_PROMPT_TOKENS, float(sum(record.prompt_tokens for record in bucket_records))),
+        (NEWRELIC_METRIC_COMPLETION_TOKENS, float(sum(record.completion_tokens for record in bucket_records))),
+        (NEWRELIC_METRIC_TOTAL_TOKENS, float(sum(record.total_tokens for record in bucket_records))),
+    )
+    count_metrics: Final[tuple[NewRelicMetric, ...]] = tuple(
+        NewRelicCountMetric(name=name, type="count", value=value, attributes=attributes) for name, value in counts
+    )
+    summary_metric: Final = NewRelicSummaryMetric(
+        name=NEWRELIC_METRIC_REQUEST_DURATION_MS,
+        type="summary",
+        value=NewRelicSummaryValue(
+            count=len(durations),
+            sum=sum(durations),
+            min=min(durations),
+            max=max(durations),
+        ),
+        attributes=attributes,
+    )
+    return (*count_metrics, summary_metric)
+
+
+def build_metric_payload(
+    records: tuple[NewRelicMetricRecord, ...],
+    *,
+    window_start: float,
+    now: float,
+) -> tuple[NewRelicMetricEnvelope, ...]:
+    """Aggregates records into one Metric API envelope for the flush window."""
+    interval_ms: Final = max(1, int((now - window_start) * 1000))
+    bucket_keys: Final = tuple(dict.fromkeys(record.bucket_key for record in records))
+    metrics: Final = tuple(
+        metric
+        for key in bucket_keys
+        for metric in _bucket_metrics(tuple(record for record in records if record.bucket_key == key))
+    )
+    common: Final[NewRelicMetricCommon] = {
+        "timestamp": int(window_start * 1000),
+        "interval.ms": interval_ms,
+    }
+    return (NewRelicMetricEnvelope(common=common, metrics=metrics),)
+
+
+class NewRelicMetricsLogger(CustomBatchLogger):
+    def __init__(
+        self,
+        newrelic_api_key: str,
+        newrelic_region: str | None = None,
+    ) -> None:
+        if not newrelic_api_key:
+            raise ValueError(
+                "newrelic_api_key is required for NewRelicMetricsLogger; "
+                "team-scoped metrics never fall back to environment credentials"
+            )
+        self.newrelic_api_key: Final = newrelic_api_key
+        self.metric_api_url: Final = resolve_newrelic_metric_endpoint(newrelic_region)
+        self.async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
+        self._stopped: bool = False
+        self._drain_lock = asyncio.Lock()
+        asyncio.create_task(self.periodic_flush())
+        self.flush_lock = asyncio.Lock()
+        super().__init__(
+            flush_lock=self.flush_lock,
+            batch_size=NEWRELIC_METRICS_MAX_BATCH_SIZE,
+            max_queue_size=NEWRELIC_METRICS_MAX_RETRY_QUEUE_SIZE,
+        )
+
+    def stop(self) -> None:
+        """Ends the periodic flush loop; called on DynamicLoggingCache eviction.
+
+        Schedules one final drain of anything still queued, so eviction never
+        silently discards records. Guarded so it can never raise into the
+        cache's eviction path.
+        """
+        self._stopped = True
+        try:
+            asyncio.get_running_loop().create_task(self._final_drain())
+        except Exception:  # noqa: BLE001  # no running loop / shutdown; the periodic loop's final drain still runs
+            verbose_logger.debug("New Relic Metrics: could not schedule final drain on stop()", exc_info=True)
+
+    async def _drain_with_retry(self) -> None:
+        """Deliver everything queued on a stopped logger, or drop it with a log.
+
+        A stopped logger has no periodic loop left, so every post-stop path
+        funnels through here. ``_drain_lock`` serializes drains: a callback that
+        appends and starts its own drain queues behind the running one instead
+        of racing it. Each pass attempts the whole current queue in
+        ``batch_size`` chunks, unlike the periodic path it does not stop at the
+        first failing chunk, so a persistently failing head never starves the
+        tail. Only after ``_MAX_DRAIN_PASSES`` against a permanently failing
+        destination is the remainder dropped, and then only the records that were
+        queued when this drain began, so every dropped record got the full retry
+        budget: a record a callback appended mid-drain is not in that snapshot,
+        so it is left for its own serialized drain rather than dropped after
+        fewer attempts, and is never stranded.
+        """
+        async with self._drain_lock:
+            attempted: Final = tuple(self.log_queue)
+            for _pass in range(NEWRELIC_METRICS_MAX_DRAIN_PASSES):
+                await self._drain_flush_once()
+                if not self.log_queue:
+                    return
+                if _pass < NEWRELIC_METRICS_MAX_DRAIN_PASSES - 1:
+                    await asyncio.sleep(2**_pass)
+            async with self.flush_lock:
+                tried_ids: Final = frozenset(id(record) for record in attempted)
+                survivors: Final = tuple(record for record in self.log_queue if id(record) not in tried_ids)
+                dropped: Final = len(self.log_queue) - len(survivors)
+                if dropped:
+                    verbose_logger.warning(
+                        "New Relic Metrics: dropping %s records after %s drain passes",
+                        dropped,
+                        NEWRELIC_METRICS_MAX_DRAIN_PASSES,
+                    )
+                self.log_queue[:] = list(survivors)  # mutable-ok: leave late arrivals for the next serialized drain
+
+    async def _drain_flush_once(self) -> None:
+        """Attempt every queued record once, in ``batch_size`` chunks, without
+        stopping at the first failing chunk so a persistently failing head does
+        not starve the tail (the periodic ``flush_queue`` deliberately stops
+        instead). Takes the queue under ``flush_lock`` and re-queues only the
+        chunks a 5xx/network error left undelivered, so records a concurrent
+        request appends during the sends survive for the next pass."""
+        async with self.flush_lock:
+            pending: Final = tuple(self.log_queue)
+            window_start: Final = self.last_flush_time
+            self.last_flush_time = time.time()
+            del self.log_queue[:]
+        if not pending:
+            return
+        chunks: Final = tuple(
+            pending[start : start + self.batch_size] for start in range(0, len(pending), self.batch_size)
+        )
+        delivered: Final = tuple([await self._classify_and_send(chunk, window_start) for chunk in chunks])
+        failed: Final = tuple(record for chunk, ok in zip(chunks, delivered) for record in (() if ok else chunk))
+        if failed:
+            self._requeue(failed)
+
+    async def _final_drain(self) -> None:
+        await self._drain_with_retry()
+
+    async def periodic_flush(self) -> None:
+        while not self._stopped:
+            await asyncio.sleep(self.flush_interval)
+            if self._stopped:
+                break
+            await self.flush_queue()
+        await self._final_drain()
+
+    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        try:
+            await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
+        except Exception as e:  # noqa: BLE001  # logging must never break the request path
+            verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None:
+        try:
+            await self._log_async_event(standard_logging_object=kwargs.get("standard_logging_object", None))
+        except Exception as e:  # noqa: BLE001  # logging must never break the request path
+            verbose_logger.exception("New Relic Metrics Layer Error - %s\n%s", e, traceback.format_exc())
+
+    async def _log_async_event(self, standard_logging_object: StandardLoggingPayload | None) -> None:
+        if standard_logging_object is None:
+            raise ValueError("standard_logging_object not found in kwargs")
+        self.log_queue.append(_metric_record_from_payload(standard_logging_object))
+        if self._stopped:
+            # A stopped logger has no periodic loop left; an in-flight callback
+            # that appends after the eviction drain delivers its own record.
+            await self._drain_with_retry()
+            return
+        if len(self.log_queue) >= self.batch_size:
+            await self.flush_queue()
+
+    async def flush_queue(self) -> None:
+        async with self.flush_lock:
+            window_start: Final = self.last_flush_time
+            self.last_flush_time = time.time()
+            queued: Final = len(self.log_queue)
+            if not queued:
+                return
+            verbose_logger.debug("New Relic Metrics: Flushing %s queued records", queued)
+            # Bounded by what is queued now: records appended mid-flush belong to
+            # the next window, and looping until empty would never end under load.
+            for _chunk in range(ceil(queued / self.batch_size)):
+                if not await self.async_send_batch(window_start=window_start):
+                    return
+
+    async def async_send_batch(self, window_start: float | None = None) -> bool:
+        """Sends the oldest ``batch_size`` records only, so a queue grown past that
+        by re-queues cannot breach the Metric API data point cap in one request.
+        Returns False once a chunk fails and is re-queued, so the caller stops."""
+        if not self.log_queue:
+            return False
+
+        batch_to_send: Final[tuple[NewRelicMetricRecord, ...]] = tuple(self.log_queue[: self.batch_size])
+        del self.log_queue[: len(batch_to_send)]
+
+        delivered: Final = await self._classify_and_send(
+            batch_to_send, window_start if window_start is not None else self.last_flush_time
+        )
+        if not delivered:
+            self._requeue(batch_to_send)
+        return delivered
+
+    async def _classify_and_send(self, batch: tuple[NewRelicMetricRecord, ...], window_start: float) -> bool:
+        """Send one chunk and classify the outcome, never touching the queue.
+        Returns True when the batch is done with (delivered on any 2xx, or a 4xx
+        a retry would only repeat, 403 being a permanent bad-key rejection), and
+        False when a 5xx or network error means the caller should re-queue it.
+
+        ``AsyncHTTPHandler.post`` raises ``HTTPStatusError`` on any non-2xx, so a
+        4xx never returns a response here; the status is read off the raised
+        error to keep the client-error path (drop) distinct from 5xx (retry)."""
+        payload: Final = build_metric_payload(records=batch, window_start=window_start, now=time.time())
+        try:
+            status = (
+                await self.async_send_compressed_data(payload)
+            ).status_code  # rebind-ok: reassigned from the raised HTTPStatusError below
+        except HTTPStatusError as e:
+            status = e.response.status_code
+        except Exception as e:  # noqa: BLE001  # transport/network failure re-queues the batch
+            verbose_logger.warning(
+                "New Relic Metrics: network error sending %s records, will retry - %s",
+                len(batch),
+                e,
+            )
+            return False
+
+        if 200 <= status < 300:
+            return True
+
+        if 400 <= status < 500 and status not in _RETRYABLE_CLIENT_STATUSES:
+            verbose_logger.warning(
+                "New Relic Metrics: %s from Metric API%s, dropping %s records.",
+                status,
+                " (permanent credential failure: invalid or revoked team ingest key)" if status == 403 else "",
+                len(batch),
+            )
+            return True
+
+        verbose_logger.warning(
+            "New Relic Metrics: %s from Metric API, will retry %s records",
+            status,
+            len(batch),
+        )
+        return False
+
+    def _requeue(self, batch: tuple[NewRelicMetricRecord, ...]) -> None:
+        """Prepends ``batch`` in place (never by assignment: records appended by
+        concurrent requests during the flush await must survive), keeping
+        chronological order so the cap drops the oldest records first."""
+        self.log_queue[:0] = batch
+        overflow: Final = len(self.log_queue) - self.max_queue_size
+        if overflow > 0:
+            del self.log_queue[:overflow]
+            verbose_logger.warning(
+                "New Relic Metrics: retry queue exceeded max_queue_size=%s; dropped %s oldest records.",
+                self.max_queue_size,
+                overflow,
+            )
+
+    async def async_send_compressed_data(self, payload: tuple[NewRelicMetricEnvelope, ...]) -> Response:
+        compressed_data: Final = gzip.compress(safe_dumps(payload).encode("utf-8"))
+        headers: Final[Mapping[str, str]] = MappingProxyType(
+            {
+                "Content-Type": "application/json",
+                "Content-Encoding": "gzip",
+                "Api-Key": self.newrelic_api_key,
+            }
+        )
+        return await self.async_client.post(
+            url=self.metric_api_url,
+            data=compressed_data,
+            headers=headers,
+        )

litellm/integrations/newrelic/newrelic_team_handler.py

@@ -0,0 +1,90 @@
+"""
+New Relic Team Handler
+
+Used to get the NewRelicMetricsLogger for a given request.
+Handles Key/Team Based New Relic metrics, following the same pattern as DataDogHandler.
+"""
+
+from typing import TYPE_CHECKING, Final
+
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.litellm_logging import StandardCallbackDynamicParams
+
+from .newrelic_metrics import NewRelicMetricsLogger
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache
+
+
+class NewRelicLoggingConfig(TypedDict):
+    newrelic_api_key: ReadOnly[str | None]
+    newrelic_region: ReadOnly[str | None]
+
+
+class NewRelicHandler:
+    @staticmethod
+    def get_newrelic_logger_for_request(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+        in_memory_dynamic_logger_cache: "DynamicLoggingCache",
+    ) -> NewRelicMetricsLogger:
+        """
+        Get a team-scoped NewRelicMetricsLogger for a given request.
+
+        Resolves and caches per-team NewRelicMetricsLogger instances using
+        DynamicLoggingCache, keyed by the team's New Relic credentials. Each unique
+        set of credentials gets its own logger instance with its own batch/flush loop.
+
+        Note: This handler is only called when a team-scoped newrelic_api_key is
+        present. The trace logger for the ``newrelic`` callback (OTel v2 / legacy
+        agent) is managed separately by _init_custom_logger_compatible_class via
+        _in_memory_loggers.
+        """
+        _credentials: Final = NewRelicHandler.get_dynamic_newrelic_logging_config(
+            standard_callback_dynamic_params=standard_callback_dynamic_params,
+        )
+
+        temp_newrelic_logger = in_memory_dynamic_logger_cache.get_cache(
+            credentials=_credentials, service_name="newrelic"
+        )
+
+        if temp_newrelic_logger is None:
+            temp_newrelic_logger = NewRelicHandler._create_newrelic_logger_from_credentials(
+                credentials=_credentials,
+                in_memory_dynamic_logger_cache=in_memory_dynamic_logger_cache,
+            )
+
+        return temp_newrelic_logger
+
+    @staticmethod
+    def _create_newrelic_logger_from_credentials(
+        credentials: NewRelicLoggingConfig,
+        in_memory_dynamic_logger_cache: "DynamicLoggingCache",
+    ) -> NewRelicMetricsLogger:
+        newrelic_logger: Final = NewRelicMetricsLogger(
+            newrelic_api_key=credentials.get("newrelic_api_key") or "",
+            newrelic_region=credentials.get("newrelic_region"),
+        )
+        in_memory_dynamic_logger_cache.set_cache(
+            credentials=credentials,
+            service_name="newrelic",
+            logging_obj=newrelic_logger,
+        )
+        verbose_logger.debug("New Relic: Created and cached new NewRelicMetricsLogger for team-scoped credentials")
+        return newrelic_logger
+
+    @staticmethod
+    def get_dynamic_newrelic_logging_config(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+    ) -> NewRelicLoggingConfig:
+        return NewRelicLoggingConfig(
+            newrelic_api_key=standard_callback_dynamic_params.get("newrelic_api_key"),
+            newrelic_region=standard_callback_dynamic_params.get("newrelic_region"),
+        )
+
+    @staticmethod
+    def _dynamic_newrelic_credentials_are_passed(
+        standard_callback_dynamic_params: StandardCallbackDynamicParams,
+    ) -> bool:
+        return standard_callback_dynamic_params.get("newrelic_api_key") is not None

litellm/integrations/opentelemetry.py

@@ -22,6 +22,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
 )
 from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.semconv import Metric
+from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.litellm_core_utils.service_tier_utils import (
@@ -1643,7 +1644,12 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
         if self._operation_duration_histogram:
             self._operation_duration_histogram.record(duration_s, attributes=common_attrs)
-            if response_obj and (usage := response_obj.get("usage")) and self._token_usage_histogram:
+            if (
+                self._token_usage_histogram
+                and response_obj
+                and not is_unbilled_non_inference_call_from_params(kwargs.get("call_type"), params, response_obj)
+                and (usage := response_obj.get("usage"))
+            ):
                 in_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "input"}
                 out_attrs: Final = {**common_attrs, TOKEN_TYPE_ATTRIBUTE: "output"}
                 self._token_usage_histogram.record(usage.get("prompt_tokens", 0), attributes=in_attrs)
@@ -1719,6 +1725,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         if not self._time_per_output_token_histogram:
             return
 
+        if is_unbilled_non_inference_call_from_params(
+            kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
+        ):
+            return
+
         # Get completion tokens from response_obj
         completion_tokens = None
         if response_obj and (usage := response_obj.get("usage")):
@@ -2049,6 +2060,26 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                 # serialise to JSON once so set_attribute never coerces.
                 guardrail_span.set_attribute("guardrail_violation_categories", safe_dumps(violation_categories))
 
+            # Billable usage counters and USD cost stamped by the provider hook
+            # (e.g. Azure Prompt Shield text records, Bedrock policy units).
+            guardrail_usage = guardrail_information.get("guardrail_usage")
+            if guardrail_usage is not None:
+                guardrail_span.set_attribute("guardrail_usage", safe_dumps(guardrail_usage))
+            guardrail_cost = guardrail_information.get("guardrail_cost")
+            if guardrail_cost is not None:
+                self.safe_set_attribute(
+                    span=guardrail_span,
+                    key="guardrail_cost",
+                    value=guardrail_cost,
+                )
+            guardrail_cost_in_spend = guardrail_information.get("guardrail_cost_in_spend")
+            if isinstance(guardrail_cost_in_spend, bool):
+                self.safe_set_attribute(
+                    span=guardrail_span,
+                    key="guardrail_cost_in_spend",
+                    value=guardrail_cost_in_spend,
+                )
+
             self._set_team_attributes_from_kwargs(guardrail_span, kwargs)
 
             guardrail_span.end(end_time=self._to_ns(end_time_datetime))
@@ -2468,7 +2499,14 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
             self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
 
-            usage: Final = response_obj and response_obj.get("usage")
+            usage: Final = (
+                response_obj.get("usage")
+                if response_obj
+                and not is_unbilled_non_inference_call_from_params(
+                    kwargs.get("call_type"), litellm_params, response_obj
+                )
+                else None
+            )
             if usage:
                 self.safe_set_attribute(
                     span=span,

litellm/integrations/otel/__init__.py

@@ -49,6 +49,7 @@ from litellm.integrations.otel.model.semconv import (
     Error,
     GenAI,
     GenAIOperation,
+    GenAIOutputType,
     GenAIProvider,
     JsonRpc,
     LiteLLM,
@@ -60,6 +61,7 @@ from litellm.integrations.otel.model.semconv import (
     RpcSystem,
     Server,
     resolve_operation,
+    resolve_output_type,
     resolve_provider,
 )
 from litellm.integrations.otel.model.spans import (
@@ -84,6 +86,7 @@ __all__ = [
     "Error",
     "GenAI",
     "GenAIOperation",
+    "GenAIOutputType",
     "GenAIProvider",
     "GuardrailSpanData",
     "JsonRpc",
@@ -116,6 +119,7 @@ __all__ = [
     "is_otel_v2_enabled",
     "promoted_baggage",
     "resolve_operation",
+    "resolve_output_type",
     "resolve_provider",
     "span_role_for_service",
     "validate_registry",

litellm/integrations/otel/emitter.py

@@ -146,7 +146,7 @@ class SpanEmitter:
         For callers that own and manage their own span lifecycle. ``tracer``
         overrides the bound tracer for this span only, used for per-request
         multi-tenant credential routing. ``links`` records related-but-not-parent
-        spans (e.g. the transport span of an MCP message, per MCP semconv).
+        spans (e.g. the trace context an MCP client propagated in ``params._meta``).
         """
         return (tracer or self._tracer).start_span(
             name,
@@ -156,6 +156,12 @@ class SpanEmitter:
             links=list(links) if links else None,
         )
 
+    def mark_emitted(self, dedup_key: str | None, role: SpanRole) -> None:
+        """Register a span emitted outside :meth:`emit` (the boundary-opened
+        LLM-call span closed via :meth:`finish_span`) so a later :meth:`emit`
+        for the same ``(dedup_key, role)`` deduplicates against it."""
+        self._seen(dedup_key, role)
+
     def _seen(self, dedup_key: str | None, role: SpanRole) -> bool:
         """Return True once a ``(dedup_key, role)`` pair has been emitted.
 
@@ -190,8 +196,8 @@ class SpanEmitter:
 
         Return the span, or ``None`` if it was deduplicated away. ``tracer``
         overrides the bound tracer for this span, used for per-request routing.
-        ``links`` records related-but-not-parent spans (the transport span of an
-        MCP message).
+        ``links`` records related-but-not-parent spans (e.g. the trace context an
+        MCP client propagated in ``params._meta``).
         """
         # LLM-call and MCP tool-call spans carry a dedup key (their request's
         # call id), so a sync+async double-firing coalesces. ``isinstance`` narrows

litellm/integrations/otel/logger.py

@@ -390,10 +390,10 @@ class OpenTelemetryV2(CustomLogger):
 
         MCP tool calls reach the success/failure callbacks like any other request
         (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have
-        no ``pre_call`` carrier — so they get their own CLIENT span here. Per the MCP
-        semconv it parents to the trace context the client propagated in
-        ``params._meta`` (or starts a new root) and links the transport span, rather
-        than nesting under the HTTP/session span. Returns whether it handled the
+        no ``pre_call`` carrier — so they get their own CLIENT span here. It nests
+        under the transport span of the request carrying this message, and trace
+        context the client propagated in ``params._meta`` is recorded as a span
+        link (see ``resolve_mcp_span_context``). Returns whether it handled the
         event, so the caller skips the LLM-call path. The whole span is emitted at
         once (there is no boundary to open it at), deduped on the call id.
         """
@@ -436,9 +436,9 @@ class OpenTelemetryV2(CustomLogger):
 
         Like a tool call, listing reaches the success/failure callbacks (here with
         ``call_type`` ``list_mcp_tools``) with no ``pre_call`` carrier, so it gets its
-        own CLIENT span. Per the MCP semconv it parents to the ``params._meta`` trace
-        context (or starts a new root) and links the transport span, rather than
-        nesting under the HTTP/session span. Returns whether it handled the event so
+        own CLIENT span, nested under the transport span of the request carrying
+        this message with any ``params._meta`` trace context recorded as a span
+        link (see ``resolve_mcp_span_context``). Returns whether it handled the event so
         the caller skips the LLM-call path.
         """
         raw_payload: Final = kwargs.get("standard_logging_object")
@@ -484,10 +484,15 @@ class OpenTelemetryV2(CustomLogger):
         # ``pop`` is the dedup: this method runs from both the success and failure
         # paths, and whichever fires first removes the carrier and closes the span.
         carrier: Final = self._open_llm_calls.pop(call_id, None) if call_id else None
-        if carrier is None:
+        # A missing carrier does not always mean nothing happened: a team/key-scoped
+        # logger is a success/failure callback only, so ``pre_call`` never reaches it
+        # and no carrier exists. The payload plus the request-level provider-handoff
+        # stamp (``upstream_started``) is the affirmative signal of a real call; a
+        # gate rejection carries ``is_no_upstream_call`` and gets no span.
+        if carrier is None and (call.is_no_upstream_call or not call.upstream_started or call.payload is None):
             return None
         try:
-            return self._finish_carrier(carrier, call, end_time)
+            return self._finish_carrier(carrier, call, start_time, end_time)
         finally:
             # After the span has ended, so a release-triggered provider shutdown
             # force-flushes it out rather than racing its enqueue.
@@ -497,8 +502,11 @@ class OpenTelemetryV2(CustomLogger):
         """Remember an in-flight LLM call, evicting the oldest if over budget.
 
         A call that opens but never closes (a stream that only fires stream
-        events) would linger otherwise; the evicted span is simply dropped
-        (never exported).
+        events) would linger otherwise. Eviction only drops the boundary carrier,
+        not the call: if that call later closes as a real completed call, it still
+        emits through the deferred branch in ``_close_llm_call`` (the same path a
+        team/key-scoped logger uses, since it never opens a carrier), deduplicated
+        by call id. Only a call that is evicted and never closes goes unexported.
         """
         self._open_llm_calls[call_id] = carrier
         if len(self._open_llm_calls) > _OPEN_CALLS_MAX:
@@ -512,15 +520,20 @@ class OpenTelemetryV2(CustomLogger):
 
     def _finish_carrier(
         self,
-        carrier: _LLMCallSpan,
+        carrier: "_LLMCallSpan | None",
         call: LLMCallEvent,
+        start_time: datetime | float | None,
         end_time: datetime | float | None,
     ) -> Span | None:
         payload: Final = call.payload
+        call_id: Final = call.call_id
         if payload is None:
-            if carrier.span is not None:
+            if carrier is not None and carrier.span is not None:
                 # Opened at the boundary but the payload never materialized — end
-                # it (named provisionally) so it isn't leaked as an open span.
+                # it (named provisionally) so it isn't leaked as an open span, and
+                # register the dedup marker so a later payload-carrying close for
+                # the same call id cannot re-emit through the deferred branch.
+                self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
                 carrier.span.end(end_time=to_ns(end_time))
             return None
         data: Final = LLMCallSpanData.from_standard_logging_payload(
@@ -529,10 +542,13 @@ class OpenTelemetryV2(CustomLogger):
             time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
         )
         end_time_ns: Final = to_ns(end_time)
-        if carrier.span is not None:
+        if carrier is not None and carrier.span is not None:
             # Born at the boundary: stamp attributes from the typed payload, set
             # status, and end it. Its parent (the server span) was captured at
-            # creation from real ambient context.
+            # creation from real ambient context. Register the dedup marker so a
+            # second close for the same call id (success then failure on one
+            # logging object) cannot re-emit through the deferred branch.
+            self._emitter.mark_emitted(call_id, SpanRole.LLM_CALL)
             self._emitter.finish_span(SpanRole.LLM_CALL, carrier.span, data, end_time_ns=end_time_ns)
             return carrier.span
         # Deferred: ``pre_call`` saw no recordable parent, so create the span now.
@@ -549,7 +565,7 @@ class OpenTelemetryV2(CustomLogger):
                 SpanRole.LLM_CALL,
                 data,
                 parent_context=(set_span_in_context(INVALID_SPAN, parent_ctx) if route.detached else parent_ctx),
-                start_time_ns=carrier.start_time_ns,
+                start_time_ns=(carrier.start_time_ns if carrier is not None else to_ns(start_time)),
                 end_time_ns=end_time_ns,
                 tracer=route.tracer,
                 links=_request_trace_links(parent_ctx) if route.detached else None,

litellm/integrations/otel/mappers/genai.py

@@ -42,6 +42,7 @@ class GenAIMapper:
     _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
         GenAI.OPERATION_NAME: lambda d: d.operation.value,
         GenAI.PROVIDER_NAME: lambda d: d.provider or None,
+        GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
         GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
         GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
         GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
@@ -65,6 +66,7 @@ class GenAIMapper:
         Server.ADDRESS: lambda d: d.server.address if d.server else None,
         Server.PORT: lambda d: d.server.port if d.server else None,
         LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
+        LiteLLM.CALL_TYPE: lambda d: d.call_type,
         # The provider/underlying model is only known once routing has picked a
         # deployment, so it can't ride identity Baggage (seeded at auth, before
         # routing) onto the boundary-born LLM span — stamp it directly here.
@@ -134,6 +136,9 @@ class GenAIMapper:
         LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
         LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
         LiteLLM.GUARDRAIL_DETECTION_METHOD: lambda d: d.detection_method,
+        LiteLLM.GUARDRAIL_USAGE: lambda d: d.usage_json,
+        LiteLLM.GUARDRAIL_COST: lambda d: d.cost,
+        LiteLLM.GUARDRAIL_COST_IN_SPEND: lambda d: d.cost_in_spend,
     }
 
     _SERVICE_ATTRS: dict[str, Callable[[ServiceSpanData], AttrValue | None]] = {

litellm/integrations/otel/model/config.py

@@ -39,6 +39,7 @@ class ExporterOwner(str, Enum):
     WEAVE_OTEL = "weave_otel"
     LEVO = "levo"
     AGENTOPS = "agentops"
+    NEWRELIC = "newrelic"
 
 
 class _OTelV2Flag(BaseSettings):
@@ -97,6 +98,15 @@ class ExporterSpec(BaseModel):
             "auto (Simple for console/in_memory, Batch otherwise)."
         ),
     )
+    requires_headers: bool = Field(
+        default=False,
+        description=(
+            "Skip this exporter when no headers are resolved. For destinations "
+            "that reject unauthenticated exports (e.g. New Relic), a spec kept "
+            "only as the per-request credential-stamping target would otherwise "
+            "export keyless traffic and produce a 4xx for every span batch."
+        ),
+    )
 
 
 class OpenTelemetryV2Config(BaseSettings):

litellm/integrations/otel/model/metadata.py

@@ -203,6 +203,11 @@ class LLMCallEvent:
     # True for synthetic proxy-gate logs (auth / rate-limit rejections): they fire
     # the ``pre_call`` hook but never made an upstream call, so they get no span.
     is_no_upstream_call: bool
+    # True once the request handed off to a provider (``pre_call`` stamped
+    # ``api_call_start_time``). The affirmative signal that an LLM call was
+    # actually attempted — router pre-call rejections, SDK failures before the
+    # provider handoff, and standalone guardrail runs all lack it.
+    upstream_started: bool
     # A best-effort ``"{operation} {model}"`` name known at ``pre_call`` time. The
     # span is renamed from the typed payload at close (``finish_span``); this only
     # needs to be reasonable for a span that never gets closed (a leak).
@@ -221,6 +226,7 @@ class LLMCallEvent:
             dynamic_params=kwargs.get("standard_callback_dynamic_params"),
             auth_metadata=auth_metadata(payload, kwargs),
             is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
+            upstream_started=kwargs.get("api_call_start_time") is not None,
             provisional_span_name=f"{operation.value} {model}".strip(),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
         )

litellm/integrations/otel/model/payloads.py

@@ -15,8 +15,10 @@ from litellm.integrations.otel.model.metadata import (
 )
 from litellm.integrations.otel.model.semconv import (
     GenAIOperation,
+    GenAIOutputType,
     MCPMethod,
     resolve_operation,
+    resolve_output_type,
     resolve_provider,
 )
 from litellm.integrations.otel.model.utils import (
@@ -188,6 +190,15 @@ class GuardrailSpanData:
     guardrail_id: str | None = None
     policy_template: str | None = None
     detection_method: str | None = None
+    # Provider-reported billable usage counters (JSON-serialized) and the USD cost
+    # priced from them by the provider hook (``guardrail_usage`` /
+    # ``guardrail_cost`` on ``StandardLoggingGuardrailInformation``).
+    usage_json: str | None = None
+    cost: float | None = None
+    # Whether ``cost`` participates in the request's billed spend (absent means
+    # billed, the default; False means report-only). Mirrors
+    # ``guardrail_cost_in_spend`` so trace consumers can avoid double-counting.
+    cost_in_spend: bool | None = None
     # Set when the guardrail intervened/blocked or failed, so the emitter marks
     # the span ERROR — a blocking guardrail is an error outcome for that span.
     error: SpanError | None = None
@@ -207,6 +218,8 @@ class GuardrailSpanData:
         get: Final = cast(Mapping[str, object], entry).get
         status: Final = as_str(get("guardrail_status"))
         response: Final = get("guardrail_response")
+        usage: Final = get("guardrail_usage")
+        in_spend: Final = get("guardrail_cost_in_spend")
         error: Final = (
             SpanError(error_type=status, message=as_str(get("guardrail_action")))
             if status in cls._ERROR_STATUSES
@@ -229,6 +242,9 @@ class GuardrailSpanData:
             guardrail_id=as_str(get("guardrail_id")),
             policy_template=as_str(get("policy_template")),
             detection_method=as_str(get("detection_method")),
+            usage_json=_json_or_none(usage) if usage is not None else None,
+            cost=as_float(get("guardrail_cost")),
+            cost_in_spend=in_spend if isinstance(in_spend, bool) else None,
             error=error,
         )
 
@@ -310,6 +326,11 @@ class LLMCallSpanData:
     choices_out: tuple[Mapping[str, object], ...] = ()
     system_fingerprint: str | None = None
     time_to_first_chunk_seconds: float | None = None
+    # The requested output modality, set only on the routes that pin one (image
+    # generation, speech, transcription, OCR), and the litellm route itself, which
+    # keeps routes the convention folds into one operation distinguishable.
+    output_type: GenAIOutputType | None = None
+    call_type: str | None = None
 
     @classmethod
     def from_standard_logging_payload(
@@ -334,8 +355,9 @@ class LLMCallSpanData:
         # otherwise the content-bearing mappers receive empty sequences and emit
         # no prompt/response text.
         finish_reasons: Final = _finish_reasons(choices_out)
+        call_type: Final = as_str(payload.get("call_type"))
         return cls(
-            operation=resolve_operation(as_str(payload.get("call_type"))),
+            operation=resolve_operation(call_type),
             provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
             request_model=context.request_model,
             response_model=context.response_model,
@@ -358,6 +380,8 @@ class LLMCallSpanData:
             choices_out=choices_out if capture_content else (),
             system_fingerprint=as_str(response.get("system_fingerprint")),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds,
+            output_type=resolve_output_type(call_type),
+            call_type=call_type or None,
         )
 
 

litellm/integrations/otel/model/semconv.py

@@ -3,7 +3,9 @@ Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anythin
 without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
 """
 
+from collections.abc import Mapping
 from enum import Enum
+from types import MappingProxyType
 from typing import Final
 
 from litellm._logging import verbose_logger
@@ -30,6 +32,22 @@ class GenAIOperation(str, Enum):
     EXECUTE_TOOL = "execute_tool"  # MCP tool-call spans
     LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
     LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
+    LITELLM_RESPONSES_MANAGEMENT = "litellm.responses_management"
+    LITELLM_MODERATION = "litellm.moderation"
+
+
+class GenAIOutputType(str, Enum):
+    """Values for ``gen_ai.output.type``, the modality the client asked for.
+
+    It is what separates the inference routes that share ``generate_content``:
+    image generation requests ``image``, speech requests ``speech``, and
+    transcription and OCR both request ``text``.
+    """
+
+    TEXT = "text"
+    JSON = "json"
+    IMAGE = "image"
+    SPEECH = "speech"
 
 
 class GenAIProvider(str, Enum):
@@ -258,6 +276,11 @@ class LiteLLM:
     """Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
 
     CALL_ID: Final = "litellm.call_id"
+    # The litellm route that produced the call. Needed because the convention maps
+    # several routes onto one operation: transcription and OCR are both
+    # ``generate_content`` with a ``text`` output type, so this is the only thing
+    # that tells them apart.
+    CALL_TYPE: Final = "litellm.call_type"
     COST_PREFIX: Final = "litellm.cost."
     METADATA_PREFIX: Final = "litellm.metadata."
     TEAM_ID: Final = "litellm.team.id"
@@ -285,6 +308,15 @@ class LiteLLM:
     GUARDRAIL_ID: Final = "litellm.guardrail.id"
     GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"
     GUARDRAIL_DETECTION_METHOD: Final = "litellm.guardrail.detection_method"
+    # Provider-reported billable usage counters, JSON-serialized into one value.
+    GUARDRAIL_USAGE: Final = "litellm.guardrail.usage"
+    # Numeric USD cost of the guardrail invocation; lives under the litellm.cost.*
+    # namespace (COST_PREFIX) beside the LLM call's litellm.cost.total.
+    GUARDRAIL_COST: Final = "litellm.cost.guardrail"
+    # Whether litellm.cost.guardrail is already inside litellm.cost.total (True,
+    # the billed default) or reported alongside it (False) — without this a trace
+    # consumer cannot tell whether adding the two double-counts.
+    GUARDRAIL_COST_IN_SPEND: Final = "litellm.guardrail.cost_in_spend"
     SERVICE_NAME: Final = "litellm.service.name"
     SERVICE_CALL_TYPE: Final = "litellm.service.call_type"
     PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms"
@@ -352,6 +384,24 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
     "aembedding": GenAIOperation.EMBEDDINGS,
     "responses": GenAIOperation.CHAT,
     "aresponses": GenAIOperation.CHAT,
+    "get_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "aget_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "delete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "adelete_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "cancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "acancel_responses": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "list_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "alist_input_items": GenAIOperation.LITELLM_RESPONSES_MANAGEMENT,
+    "image_generation": GenAIOperation.GENERATE_CONTENT,
+    "aimage_generation": GenAIOperation.GENERATE_CONTENT,
+    "moderation": GenAIOperation.LITELLM_MODERATION,
+    "amoderation": GenAIOperation.LITELLM_MODERATION,
+    "ocr": GenAIOperation.GENERATE_CONTENT,
+    "aocr": GenAIOperation.GENERATE_CONTENT,
+    "speech": GenAIOperation.GENERATE_CONTENT,
+    "aspeech": GenAIOperation.GENERATE_CONTENT,
+    "transcription": GenAIOperation.GENERATE_CONTENT,
+    "atranscription": GenAIOperation.GENERATE_CONTENT,
     "call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
     "vector_store_search": GenAIOperation.RETRIEVAL,
     "avector_store_search": GenAIOperation.RETRIEVAL,
@@ -385,6 +435,23 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
 }
 
 
+# litellm ``call_type`` -> ``gen_ai.output.type``. Only the call types whose route
+# fixes the requested modality are listed; the attribute is conditionally required
+# on a request that asks for an output format, so anything else is left unstamped.
+_OUTPUT_TYPE_BY_CALL_TYPE: Final[Mapping[str, GenAIOutputType]] = MappingProxyType(
+    {
+        "image_generation": GenAIOutputType.IMAGE,
+        "aimage_generation": GenAIOutputType.IMAGE,
+        "speech": GenAIOutputType.SPEECH,
+        "aspeech": GenAIOutputType.SPEECH,
+        "transcription": GenAIOutputType.TEXT,
+        "atranscription": GenAIOutputType.TEXT,
+        "ocr": GenAIOutputType.TEXT,
+        "aocr": GenAIOutputType.TEXT,
+    }
+)
+
+
 def resolve_provider(custom_llm_provider: str | None) -> str:
     """Map a litellm provider string to a ``gen_ai.provider.name`` value.
 
@@ -416,3 +483,11 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
         GenAIOperation.CHAT.value,
     )
     return GenAIOperation.CHAT
+
+
+def resolve_output_type(call_type: str | None) -> GenAIOutputType | None:
+    """Map a litellm ``call_type`` to a ``gen_ai.output.type`` value, or ``None``
+    for a route that doesn't pin the output modality."""
+    if not call_type:
+        return None
+    return _OUTPUT_TYPE_BY_CALL_TYPE.get(call_type.lower())

litellm/integrations/otel/model/spans.py

@@ -10,6 +10,8 @@ Canonical hierarchy::
     │   └── DB_CALL (CLIENT)          #   its key/user/team lookups nest here
     ├── GUARDRAIL  (INTERNAL)         # request-lifecycle hook, sibling of LLM_CALL
     ├── LLM_CALL   (CLIENT)
+    ├── MCP_TOOL_CALL  (CLIENT)       # nests under the POST carrying the message
+    ├── MCP_LIST_TOOLS (CLIENT)       #   (client-propagated context is a span link)
     └── DB_CALL    (CLIENT)           # e.g. the spend-log write
 
 Guardrails parent to PROXY_REQUEST, not LLM_CALL: pre/during/post-call guardrail
@@ -18,14 +20,14 @@ before the LLM call even starts), so a guardrail is a sibling of the LLM call,
 not a child of it. The emitter parents every span to the ambient OTel context
 (the active server span), which matches this.
 
-MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) have two shapes, chosen at emit
-time by :func:`resolve_mcp_span_context`. When the client propagates trace context
-in ``params._meta`` MCP and the HTTP transport are independent contexts per the
-OTel GenAI MCP semconv, so the span parents to that propagated context and records
-the ``PROXY_REQUEST`` transport span as a span *link*, never a parent — the shape
-this registry's ``parent=None, links=PROXY_REQUEST`` entry encodes. When nothing is
-propagated (the common case) the span nests under the transport span of the request
-carrying that message, so the tool call stays in one trace.
+MCP spans (``MCP_TOOL_CALL``, ``MCP_LIST_TOOLS``) are parented at emit time by
+:func:`resolve_mcp_span_context`: they nest under the ``PROXY_REQUEST`` transport
+span of the request carrying that message, so the tool call stays in one trace.
+Trace context the client propagated in ``params._meta`` (SEP-414) is recorded as
+a span *link*, never the parent — a remote parent would root the span in a trace
+whose root never reaches the gateway's tracing backend. Links always target that
+remote client context, never a registry role, so ``SpanSpec`` declares no link
+field; the concrete transport parent is resolved per message at emit time.
 
 Not every service call becomes a span — :func:`span_role_for_service` decides:
 
@@ -85,25 +87,19 @@ class SpanSpec:
     role: SpanRole
     kind: LiteLLMSpanKind
     parent: SpanRole | None
-    links: SpanRole | None = None
 
 
 SPAN_REGISTRY: Final[dict[SpanRole, SpanSpec]] = {
     SpanRole.PROXY_REQUEST: SpanSpec(SpanRole.PROXY_REQUEST, LiteLLMSpanKind.SERVER, parent=None),
     SpanRole.LLM_CALL: SpanSpec(SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     # The proxy is an MCP client to the upstream server, so MCP spans are CLIENT
-    # spans. With trace context propagated in ``params._meta``, MCP and the HTTP
-    # transport are independent contexts (OTel GenAI MCP semconv): the span parents
-    # to the propagated context and records the PROXY_REQUEST transport span as a
-    # span *link*, never a parent — the shape ``parent=None, links=PROXY_REQUEST``
-    # encodes. With nothing propagated, ``resolve_mcp_span_context`` nests the span
-    # under that message's transport span instead, keeping the call in one trace.
-    SpanRole.MCP_TOOL_CALL: SpanSpec(
-        SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
-    ),
-    SpanRole.MCP_LIST_TOOLS: SpanSpec(
-        SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=None, links=SpanRole.PROXY_REQUEST
-    ),
+    # spans. ``resolve_mcp_span_context`` nests them under the PROXY_REQUEST
+    # transport span of the request carrying that message (resolved per message at
+    # emit time), keeping the call in one trace. Trace context the client
+    # propagated in ``params._meta`` becomes a span *link* to that remote context,
+    # which is not a registry role, so ``SpanSpec`` has no link field.
+    SpanRole.MCP_TOOL_CALL: SpanSpec(SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
+    SpanRole.MCP_LIST_TOOLS: SpanSpec(SpanRole.MCP_LIST_TOOLS, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     SpanRole.GUARDRAIL: SpanSpec(SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
     SpanRole.DB_CALL: SpanSpec(SpanRole.DB_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST),
     SpanRole.SERVICE: SpanSpec(SpanRole.SERVICE, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST),
@@ -209,8 +205,8 @@ def service_span_name(data: "ServiceSpanData") -> str:
 
 
 def root_roles() -> list[SpanRole]:
-    """Roles with no in-process parent. They start a new trace unless they adopt a
-    remote parent (e.g. an MCP span joining the client's propagated context)."""
+    """Roles with no in-process parent, i.e. they start a new trace (only the
+    instrumentor-owned ``PROXY_REQUEST`` server span today)."""
     return [role for role, spec in SPAN_REGISTRY.items() if spec.parent is None]
 
 
@@ -227,8 +223,6 @@ def validate_registry(
             raise ValueError(f"SPAN_REGISTRY[{role}] has mismatched role {spec.role}")
         if spec.parent is not None and spec.parent not in reg:
             raise ValueError(f"span role {role} declares unknown parent {spec.parent}")
-        if spec.links is not None and spec.links not in reg:
-            raise ValueError(f"span role {role} declares unknown link target {spec.links}")
     missing: Final = [role for role in SpanRole if role not in reg]
     if missing:
         raise ValueError(f"SPAN_REGISTRY is missing roles: {missing}")

litellm/integrations/otel/plumbing/context.py

@@ -57,8 +57,8 @@ def request_root_span() -> "Span | None":
 
 # The W3C trace-context carrier (``traceparent``/``tracestate``/``baggage``) the
 # MCP client propagated in the current request's ``params._meta``. The MCP gateway
-# sets it per message so the MCP span can parent to the client's span rather than
-# to the transport. A ``ContextVar`` because, like the root-span anchor, it must
+# sets it per message so the MCP span can record the client's span as a span
+# link. A ``ContextVar`` because, like the root-span anchor, it must
 # ride the request task and be readable by the inline success-logging callback.
 _mcp_message_trace_carrier: Final["ContextVar[Mapping[str, str] | None]"] = ContextVar(
     "litellm_otel_mcp_message_trace_carrier", default=None
@@ -148,10 +148,10 @@ def _mcp_transport_span_context() -> "SpanContext | None":
 
     Prefers the transport the gateway published for this specific message; falls
     back to the ambient request anchor for paths that emit an MCP span on the
-    request task itself (the REST MCP endpoints, the SDK). Parenting and linking
-    only need the immutable context, and unlike ``mcp_message_transport_span`` they
-    stay correct against a transport that has already finished, so this does not
-    require the span to still be recording.
+    request task itself (the REST MCP endpoints). Parenting needs only the
+    immutable context, and unlike ``mcp_message_transport_span`` it stays correct
+    against a transport that has already finished, so this does not require the
+    span to still be recording.
     """
     published: Final = _mcp_message_transport_span.get()
     if published is not None:
@@ -222,25 +222,31 @@ def resolve_mcp_span_context(
 ) -> "tuple[Context, tuple[Link, ...]]":
     """Parent context + links for an MCP message span.
 
+    The span always nests under the transport span of the request carrying this
+    message, so a tool call and the ``POST`` that carried it stay in one trace.
+    The transport comes from :func:`_mcp_transport_span_context`, which is the
+    *current message's* POST rather than whatever request happened to open the
+    session, so a long-lived session does not glue every message under its first
+    request.
+
     When the client propagates W3C trace context in the request's ``params._meta``
-    (SEP-414), MCP and the underlying transport are independent lifecycles — one
-    streamable-HTTP session multiplexes many messages, and the client's own span is
-    the truthful parent. So, per the OTel GenAI MCP semconv:
+    (SEP-414), that remote context is recorded as a span *link*, never the parent.
+    The OTel GenAI MCP semconv prefers the inverse (remote parent, transport link),
+    but the gateway's tracing backend only ever receives the gateway's half of such
+    a trace: parenting into the client's trace id roots the span in a trace whose
+    root span never reaches the backend, so the span is unreachable from the trace
+    view and the transport transaction shows a dangling link (observed with
+    clients that propagate synthetic trace ids). Anchoring to the gateway's own
+    request and linking the client's context keeps every trace renderable while
+    preserving the client-side correlation.
 
-    * parent to the trace context the client propagated (a *remote* parent), and
-    * record the transport span as a *link*, never the parent.
-
-    Almost no client implements SEP-414 yet, so in practice nothing is propagated.
-    Rooting the span there splits a single tool call into two disconnected traces
-    joined only by a link, which is how it surfaces in APM: the ``POST`` transaction
-    and the ``tools/call`` span share no trace. With no remote parent to honor,
-    parent to the transport span of the request carrying this message instead, so
-    the call stays in one trace; no link is added since the transport is now the
-    real parent. The transport comes from :func:`_mcp_transport_span_context`, which
-    is the *current message's* POST rather than whatever request happened to open
-    the session, so a long-lived session does not glue every message under its
-    first request. With neither a remote parent nor a transport the returned context
-    carries no span and the span legitimately starts its own root trace.
+    With no transport at all the span starts its own root trace, still carrying
+    the link — the client context is only ever a link, so this event keeps one
+    shape everywhere. Both returned contexts are built on an explicitly empty
+    base, so ambient (stale session) state can never leak in, and the span
+    inherits the transport's sampling decision exactly like every other
+    request-level span — a client's sampled flag neither forces nor suppresses
+    recording.
 
     Only trace context (``traceparent``/``tracestate``) is extracted, never the
     client's W3C Baggage: ``params._meta`` is caller-controlled, and the otel
@@ -251,13 +257,12 @@ def resolve_mcp_span_context(
     never fall through to the ambient (stale session) span.
     """
     source: Final = carrier if carrier is not None else _mcp_message_trace_carrier.get()
-    parent: Final = _PROPAGATOR.extract(dict(source or {}), context=Context())
+    propagated: Final = get_current_span(_PROPAGATOR.extract(dict(source or {}), context=Context()))
+    links: Final = (Link(propagated.get_span_context()),) if is_recordable_span(propagated) else ()
     transport: Final = _mcp_transport_span_context()
-    if is_recordable_span(get_current_span(parent)):
-        return parent, (Link(transport),) if transport is not None else ()
-    if transport is not None:
-        return context_from_span(NonRecordingSpan(transport)), ()
-    return parent, ()
+    if transport is None:
+        return Context(), links
+    return context_from_span(NonRecordingSpan(transport), context=Context()), links
 
 
 def is_recordable_span(obj: object) -> bool:

litellm/integrations/otel/plumbing/metrics.py

@@ -32,6 +32,7 @@ from litellm.integrations.otel.model.semconv import (
     resolve_provider,
 )
 from litellm.integrations.otel.model.utils import to_seconds
+from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
 
@@ -198,16 +199,21 @@ class GenAIMetricRecorder:
     ) -> None:
         common_attrs: Final = self._filter_attributes(self._bounded_attributes(kwargs))
         duration_s: Final = (end_time - start_time).total_seconds()
+        usage_is_replayed: Final = is_unbilled_non_inference_call_from_params(
+            kwargs.get("call_type"), kwargs.get("litellm_params"), response_obj
+        )
 
         self._metrics.operation_duration.record(duration_s, attributes=common_attrs)
-        self._record_token_usage(response_obj, common_attrs)
+        if not usage_is_replayed:
+            self._record_token_usage(response_obj, common_attrs)
 
         cost: Final = kwargs.get("response_cost")
         if cost:
             self._metrics.token_cost.record(cost, attributes=common_attrs)
 
         self._record_time_to_first_token(kwargs, common_attrs)
-        self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
+        if not usage_is_replayed:
+            self._record_time_per_output_token(kwargs, response_obj, end_time, duration_s, common_attrs)
         self._record_response_duration(kwargs, end_time, common_attrs)
 
     def record_failure(

litellm/integrations/otel/plumbing/providers.py

@@ -436,6 +436,8 @@ def build_tracer_provider(
     # ``config._normalize`` guarantees at least one spec (it folds the top-level
     # ``exporter``/``endpoint``/``headers`` fields in when ``exporters`` is empty).
     for spec in config.exporters:
+        if spec.requires_headers and not spec.headers:
+            continue
         exp = _exporter_from_spec(spec)
         provider.add_span_processor(
             _processor_for(

litellm/integrations/otel/plumbing/routing.py

@@ -2,12 +2,13 @@
 
 When a request carries team/key vendor credentials in
 ``standard_callback_dynamic_params``, or the key/team config resolved at auth
-names a destination project, its spans must export through a
-``TracerProvider`` whose OTLP headers carry those credentials / that project.
-``TenantTracerCache`` builds and caches one provider per distinct
-(credentials, project) pair, and otherwise hands back the logger's default
-tracer. This lets a single logger fan requests out to many tenants without
-needing a logger per tenant.
+names a destination project or a service name, its spans must export through a
+``TracerProvider`` whose OTLP headers carry those credentials / that project,
+or whose Resource carries that ``service.name``. ``TenantTracerCache`` builds
+and caches one provider per distinct (credentials, project, service name)
+tuple, and otherwise hands back the logger's default tracer. This lets a
+single logger fan requests out to many tenants without needing a logger per
+tenant.
 """
 
 import threading
@@ -15,22 +16,25 @@ from collections import OrderedDict
 from collections.abc import Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import Any, Final, TypeAlias
+from typing import Final, TypeAlias
 from urllib.parse import quote
 
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.trace import Tracer
 
 from litellm._logging import verbose_logger
+from litellm.constants import OTEL_SERVICE_NAME_METADATA_KEYS
 from litellm.integrations.otel.model.config import ExporterSpec, OpenTelemetryV2Config
 from litellm.integrations.otel.plumbing.providers import (
     build_tracer_provider,
     get_tracer,
 )
 from litellm.integrations.otel.presets import (
+    dynamic_otlp_endpoint,
     dynamic_otlp_headers,
     project_routing_headers,
 )
+from litellm.types.utils import StandardCallbackDynamicParams
 
 # Exporter kinds that ignore headers — never rewritten with dynamic credentials.
 _NON_OTLP_KINDS: Final = ("console", "in_memory", "inmemory", "memory")
@@ -63,8 +67,30 @@ _MAX_RETIRED_PROVIDERS: Final = 64
 
 _HeaderItems: TypeAlias = tuple[tuple[str, str], ...]
 
+_RouteKey: TypeAlias = tuple[_HeaderItems, _HeaderItems, str | None, str | None]
+
 _NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})
 
+#: Key/team config fields naming the Resource ``service.name``, highest
+#: precedence first. Read only from ``user_api_key_auth_metadata`` (the config
+#: the proxy resolved at auth), never from client-supplied request metadata:
+#: the service name picks the dataset/service traces land in (Honeycomb routes
+#: datasets by it), so a caller must not be able to choose one.
+_SERVICE_NAME_KEYS: Final = OTEL_SERVICE_NAME_METADATA_KEYS
+
+
+def tenant_service_name(auth_metadata: Mapping[str, str] | None) -> str | None:
+    """The per-request ``service.name`` override for this key/team, if any.
+
+    ``None`` keeps the env-configured default (``OTEL_SERVICE_NAME``).
+    """
+    if not auth_metadata:
+        return None
+    return next(
+        (stripped for key in _SERVICE_NAME_KEYS if (stripped := (auth_metadata.get(key) or "").strip())),
+        None,
+    )
+
 
 def _shutdown_provider(provider: TracerProvider) -> None:
     """Flush + stop an evicted provider's processors (reclaims their threads).
@@ -114,7 +140,7 @@ class TenantRoute:
 
 
 class TenantTracerCache:
-    """Credential/project-scoped ``TracerProvider`` cache keyed by the routing headers."""
+    """Tenant-scoped ``TracerProvider`` cache keyed by routing headers and service name."""
 
     def __init__(
         self,
@@ -129,7 +155,9 @@ class TenantTracerCache:
         # thread-pool workers concurrently with the event loop, so cache
         # updates, span counts, and retirement must be atomic.
         self._lock: Final = threading.Lock()
-        self._providers: OrderedDict[tuple[_HeaderItems, _HeaderItems], TracerProvider] = OrderedDict()
+        self._providers: OrderedDict[_RouteKey, TracerProvider] = (
+            OrderedDict()  # mutable-ok: bounded LRU; eviction needs in-place ordered mutation
+        )
         self._open_span_counts: dict[TracerProvider, int] = {}  # mutable-ok: live refcount state
         # Oldest-first so an overflow of draining providers sheds the stalest.
         self._retired: OrderedDict[TracerProvider, None] = OrderedDict()  # mutable-ok: draining evicted providers
@@ -163,15 +191,16 @@ class TenantTracerCache:
     def route_for(
         self,
         default: Tracer,
-        dynamic_params: Any,
+        dynamic_params: StandardCallbackDynamicParams | None,
         auth_metadata: Mapping[str, str] | None = None,
     ) -> TenantRoute:
         """Return the tracer (and trace-detachment flag) for this request.
 
-        Use ``default`` unless the request's dynamic credentials or its key/team
-        project require a scoped tracer, in which case build (or reuse) one. The
-        cache is a bounded LRU: the least-recently-used provider is flushed and
-        shut down on overflow so its exporter threads don't accumulate.
+        Use ``default`` unless the request's dynamic credentials, its key/team
+        project, or its key/team service name require a scoped tracer, in
+        which case build (or reuse) one. The cache is a bounded LRU: the
+        least-recently-used provider is flushed and shut down on overflow so
+        its exporter threads don't accumulate.
 
         A routed provider is returned already held — its open-span count is
         incremented in the same critical section as the cache update — so a
@@ -180,14 +209,22 @@ class TenantTracerCache:
         """
         credential_headers: Final = dynamic_otlp_headers(self._callback_name, dynamic_params) or _NO_HEADERS
         project_headers: Final = self._project_headers(auth_metadata)
-        if not credential_headers and not project_headers:
+        service_name: Final = tenant_service_name(auth_metadata)
+        if not credential_headers and not project_headers and service_name is None:
             return TenantRoute(tracer=default, detached=False)
+        # A fixed per-integration region endpoint (New Relic us/eu), never a
+        # caller-supplied host; ``None`` keeps the preset's own endpoint.
+        endpoint: Final = dynamic_otlp_endpoint(self._callback_name, dynamic_params)
         cache_key: Final = (
             tuple(sorted(credential_headers.items())),
             tuple(sorted(project_headers.items())),
+            endpoint,
+            service_name,
         )
         with self._lock:
-            provider: Final = self._cached_provider_locked(cache_key, credential_headers, project_headers)
+            provider: Final = self._cached_provider_locked(
+                cache_key, credential_headers, project_headers, endpoint, service_name
+            )
             self._open_span_counts[provider] = self._open_span_counts.get(provider, 0) + 1
             evicted: Final = self._evicted_on_overflow_locked()
         if evicted is not None:
@@ -200,15 +237,19 @@ class TenantTracerCache:
 
     def _cached_provider_locked(
         self,
-        cache_key: tuple[_HeaderItems, _HeaderItems],
+        cache_key: _RouteKey,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None,
+        service_name: str | None,
     ) -> TracerProvider:
         cached: Final = self._providers.get(cache_key)
         if cached is not None:
             self._providers.move_to_end(cache_key)
             return cached
-        built: Final = build_tracer_provider(self._routed_config(credential_headers, project_headers))
+        built: Final = build_tracer_provider(
+            self._routed_config(credential_headers, project_headers, endpoint, service_name)
+        )
         self._providers[cache_key] = built
         return built
 
@@ -257,6 +298,8 @@ class TenantTracerCache:
         self,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None = None,
+        service_name: str | None = None,
     ) -> OpenTelemetryV2Config:
         """Clone the config, rewriting headers on the callback's own exporter.
 
@@ -272,15 +315,20 @@ class TenantTracerCache:
         ``Authorization``), which must survive routing to a project.
         """
         exporters: Final = [
-            self._routed_exporter(spec, credential_headers, project_headers) for spec in self._config.exporters
+            self._routed_exporter(spec, credential_headers, project_headers, endpoint)
+            for spec in self._config.exporters
         ]
-        return self._config.model_copy(update={"exporters": exporters})
+        update: Final = (
+            {"exporters": exporters} if service_name is None else {"exporters": exporters, "service_name": service_name}
+        )
+        return self._config.model_copy(update=update)
 
     def _routed_exporter(
         self,
         spec: ExporterSpec,
         credential_headers: Mapping[str, str],
         project_headers: Mapping[str, str],
+        endpoint: str | None = None,
     ) -> ExporterSpec:
         kind: Final = spec.kind.lower()
         if spec.owner != self._callback_name or kind in _NON_OTLP_KINDS:
@@ -291,4 +339,10 @@ class TenantTracerCache:
             if project_headers and kind not in _GRPC_KINDS
             else base
         )
-        return spec if routed == spec.headers else spec.model_copy(update={"headers": routed})
+        update: Final = {  # mutable-ok: model_copy(update=...) requires a plain dict
+            field: value
+            for field, value in (("headers", routed), ("endpoint", endpoint))
+            if (field == "headers" and routed != spec.headers)
+            or (field == "endpoint" and endpoint is not None and endpoint != spec.endpoint)
+        }
+        return spec if not update else spec.model_copy(update=update)

litellm/integrations/otel/presets/__init__.py

@@ -21,6 +21,11 @@ from litellm.integrations.otel.presets.langfuse import (
 )
 from litellm.integrations.otel.presets.langtrace import langtrace_preset
 from litellm.integrations.otel.presets.levo import levo_preset
+from litellm.integrations.otel.presets.newrelic import (
+    newrelic_dynamic_endpoint,
+    newrelic_dynamic_headers,
+    newrelic_preset,
+)
 from litellm.integrations.otel.presets.phoenix import (
     phoenix_preset,
     phoenix_project_headers,
@@ -30,25 +35,45 @@ from litellm.types.utils import StandardCallbackDynamicParams
 
 #: Callback name → preset. The ``Preset`` annotation makes mypy verify every
 #: registered value matches the preset interface.
-PRESET_BY_CALLBACK: Final[dict[str, Preset]] = {
-    "agentops": agentops_preset,
-    "arize": arize_preset,
-    "arize_phoenix": phoenix_preset,
-    "langfuse_otel": langfuse_preset,
-    "langtrace": langtrace_preset,
-    "levo": levo_preset,
-    "weave_otel": weave_preset,
-}
+PRESET_BY_CALLBACK: Final[Mapping[str, Preset]] = MappingProxyType(
+    {
+        "agentops": agentops_preset,
+        "arize": arize_preset,
+        "arize_phoenix": phoenix_preset,
+        "langfuse_otel": langfuse_preset,
+        "langtrace": langtrace_preset,
+        "levo": levo_preset,
+        "newrelic": newrelic_preset,
+        "weave_otel": weave_preset,
+    }
+)
 
 #: Callback name → per-request OTLP header builder (team/key multi-tenant
 #: routing). Only integrations that support dynamic credentials appear here —
 #: Arize-Phoenix/Langtrace/Levo/AgentOps don't, so they use the logger's
 #: default tracer.
-DYNAMIC_HEADERS_BY_CALLBACK: Final[dict[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = {
-    "arize": arize_dynamic_headers,
-    "langfuse_otel": langfuse_dynamic_headers,
-    "weave_otel": weave_dynamic_headers,
-}
+DYNAMIC_HEADERS_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], dict[str, str]]]] = (
+    MappingProxyType(
+        {
+            "arize": arize_dynamic_headers,
+            "langfuse_otel": langfuse_dynamic_headers,
+            "newrelic": newrelic_dynamic_headers,
+            "weave_otel": weave_dynamic_headers,
+        }
+    )
+)
+
+#: Callback name → per-request OTLP endpoint resolver. Only integrations whose
+#: destination host varies per tenant (from a fixed region table, never a
+#: caller-supplied URL) appear here; for everyone else the preset's endpoint is
+#: authoritative.
+DYNAMIC_ENDPOINT_BY_CALLBACK: Final[Mapping[str, Callable[[StandardCallbackDynamicParams], str | None]]] = (
+    MappingProxyType(
+        {
+            "newrelic": newrelic_dynamic_endpoint,
+        }
+    )
+)
 
 
 #: Callback name → per-request *routing* header builder, sourced from the key/team
@@ -98,17 +123,34 @@ def project_routing_headers(
     return builder(auth_metadata)
 
 
+def dynamic_otlp_endpoint(
+    callback_name: str | None,
+    dynamic_params: StandardCallbackDynamicParams | None,
+) -> str | None:
+    """Per-request OTLP endpoint for ``callback_name``, or ``None`` if N/A.
+
+    ``None`` means "keep the preset's own endpoint".
+    """
+    resolver: Final = DYNAMIC_ENDPOINT_BY_CALLBACK.get(callback_name or "")
+    if resolver is None or not dynamic_params:
+        return None
+    return resolver(dynamic_params)
+
+
 __all__ = [
+    "DYNAMIC_ENDPOINT_BY_CALLBACK",
     "DYNAMIC_HEADERS_BY_CALLBACK",
     "PRESET_BY_CALLBACK",
     "PROJECT_HEADERS_BY_CALLBACK",
     "Preset",
     "agentops_preset",
     "arize_preset",
+    "dynamic_otlp_endpoint",
     "dynamic_otlp_headers",
     "langfuse_preset",
     "langtrace_preset",
     "levo_preset",
+    "newrelic_preset",
     "phoenix_preset",
     "project_routing_headers",
     "weave_preset",

litellm/integrations/otel/presets/newrelic.py

@@ -0,0 +1,104 @@
+"""New Relic preset — OTLP/HTTP exporter to New Relic + GenAI vocabulary."""
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from litellm._logging import verbose_logger
+from litellm.integrations.otel.model.config import (
+    ExporterOwner,
+    ExporterSpec,
+    OpenTelemetryV2Config,
+)
+from litellm.integrations.otel.presets.utils import ensure_mappers
+from litellm.types.utils import StandardCallbackDynamicParams
+
+#: Region -> OTLP base endpoint. A fixed table by design: team config picks a
+#: region enum rather than a free-form endpoint, so callback vars can never
+#: redirect telemetry to an arbitrary host.
+NEWRELIC_OTLP_ENDPOINT_BY_REGION: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "us": "https://otlp.nr-data.net",
+        "eu": "https://otlp.eu01.nr-data.net",
+    }
+)
+
+_DEFAULT_REGION: Final = "us"
+
+
+class _NewRelicSettings(BaseSettings):
+    model_config = SettingsConfigDict(case_sensitive=False, extra="ignore")
+
+    # The same env vars the agent-based integration documents; the key is the
+    # operator-level fallback for traffic without team credentials, the region
+    # picks that fallback's data center, and the record-content flag keeps its
+    # documented meaning when the OTel path replaces the agent.
+    license_key: str | None = Field(default=None, validation_alias="NEW_RELIC_LICENSE_KEY")
+    region: str | None = Field(default=None, validation_alias="NEW_RELIC_REGION")
+    record_content: bool | None = Field(default=None, validation_alias="NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED")
+
+
+def newrelic_preset(
+    *,
+    config_overrides: OpenTelemetryV2Config | None = None,
+) -> OpenTelemetryV2Config:
+    settings: Final = _NewRelicSettings()
+    base: Final = config_overrides or OpenTelemetryV2Config()
+    endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(
+        (settings.region or _DEFAULT_REGION).lower(), NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
+    )
+    return base.model_copy(
+        update={
+            "exporters": [
+                *base.exporters,
+                ExporterSpec(
+                    kind="otlp_http",
+                    endpoint=endpoint,
+                    headers=(f"api-key={settings.license_key}" if settings.license_key else None),
+                    owner=ExporterOwner.NEWRELIC,
+                    requires_headers=True,
+                ),
+            ],
+            # New Relic ingests the OTLP GenAI semantic conventions natively.
+            "mapper_names": ensure_mappers(base.mapper_names, "genai"),
+            **(
+                {"capture_message_content": ("span_only" if settings.record_content else "no_content")}
+                if settings.record_content is not None
+                else {}
+            ),
+        }
+    )
+
+
+def newrelic_dynamic_headers(params: StandardCallbackDynamicParams) -> dict[str, str]:
+    """Per-request New Relic OTLP headers from team/key dynamic params."""
+    api_key: Final = params.get("newrelic_api_key")
+    return {header: value for header, value in (("api-key", api_key),) if value}
+
+
+def newrelic_dynamic_endpoint(params: StandardCallbackDynamicParams) -> str:
+    """Per-request OTLP endpoint for the team's ``newrelic_region``.
+
+    Always the team's own region endpoint, defaulting to US when the team left
+    the region unset. It never falls through to the preset's endpoint, which
+    follows the operator's ``NEW_RELIC_REGION`` env; a team that saved only its
+    ingest key must not inherit the operator's region and have its US-account
+    spans rejected by an EU-configured default (or vice versa). An unknown
+    region likewise resolves to the documented US default rather than a guess.
+    """
+    region: Final = params.get("newrelic_region")
+    default_endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION[_DEFAULT_REGION]
+    if not region:
+        return default_endpoint
+    endpoint: Final = NEWRELIC_OTLP_ENDPOINT_BY_REGION.get(region.lower())
+    if endpoint is None:
+        verbose_logger.warning(
+            "New Relic: unknown newrelic_region %r; supported regions: %s. Using the default (US) endpoint.",
+            region,
+            ", ".join(sorted(NEWRELIC_OTLP_ENDPOINT_BY_REGION)),
+        )
+        return default_endpoint
+    return endpoint

litellm/integrations/prometheus.py

@@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
     _sanitize_prometheus_label_name,
     _sanitize_prometheus_label_value,
+    validate_prometheus_deployment_and_latency_caller_identity,
 )
 from litellm.types.utils import (
     StandardLoggingGuardrailInformation,
@@ -96,7 +97,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
 
 def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
     """View a repository's prisma table through the pagination surface budget metrics need."""
-    return repository.table
+    return cast(
+        _PaginatedPrismaTable[_TableRowT],
+        repository.table,  # cast-ok: prisma rows carry the budget columns the domain model declares
+    )
 
 
 class _OrgBudgetRow(Protocol):
@@ -172,6 +176,11 @@ class PrometheusLogger(CustomLogger):
         try:
             from prometheus_client import Counter, Gauge, Histogram
 
+            # Validate the caller-identity mode before any collector registers so an
+            # invalid value cannot leave partially-registered metrics behind in the
+            # process-global registry.
+            validate_prometheus_deployment_and_latency_caller_identity()
+
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
@@ -215,7 +224,9 @@ class PrometheusLogger(CustomLogger):
             # request latency metrics
             self.litellm_request_total_latency_metric = self._histogram_factory(
                 "litellm_request_total_latency_metric",
-                "Total latency (seconds) for a request to LiteLLM",
+                "End-to-end latency (seconds) for a request to LiteLLM Proxy Server, from the moment "
+                "the request reached the proxy through the end of processing -- includes "
+                "authentication, pre-call hooks, the LLM API call, and post-call processing",
                 labelnames=self.get_labels_for_metric("litellm_request_total_latency_metric"),
                 buckets=self.latency_buckets,
             )
@@ -458,7 +469,8 @@ class PrometheusLogger(CustomLogger):
             # Request queue time metric
             self.litellm_request_queue_time_metric = self._histogram_factory(
                 "litellm_request_queue_time_seconds",
-                "Time spent in request queue before processing starts (seconds)",
+                "Time (seconds) from request arrival at the proxy to the start of pre-call "
+                "processing -- includes authentication and any ASGI-level queueing",
                 labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
                 buckets=self.latency_buckets,
             )
@@ -2078,27 +2090,37 @@ class PrometheusLogger(CustomLogger):
                 _labels,
             )
 
-        # total request latency
+        # request queue time (time from arrival to processing start) -- read first so
+        # it can be folded into the total-latency metric below. start_time/end_time
+        # only span from after auth completes, so without this the "total" latency
+        # metric silently excludes auth and pre-call hook time.
+        _litellm_params: Final = kwargs.get("litellm_params", {}) or {}
+        queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
+
+        # total request latency: true end-to-end, from request arrival (queue_time_seconds,
+        # when available) through the end of processing.
         total_time_seconds: Final = self._safe_duration_seconds(
             start_time=start_time,
             end_time=end_time,
         )
         if total_time_seconds is not None:
+            _observed_total_time_seconds: Final = (
+                total_time_seconds + queue_time_seconds
+                if queue_time_seconds is not None and queue_time_seconds >= 0
+                else total_time_seconds
+            )
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"),
                 enum_values=enum_values,
                 label_context=label_context,
             )
-            self.litellm_request_total_latency_metric.labels(**_labels).observe(total_time_seconds)
+            self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds)
             self._track_end_user_metric_series(
                 self.litellm_request_total_latency_metric,
                 "litellm_request_total_latency_metric",
                 _labels,
             )
 
-        # request queue time (time from arrival to processing start)
-        _litellm_params: Final = kwargs.get("litellm_params", {}) or {}
-        queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
         if queue_time_seconds is not None and queue_time_seconds >= 0:
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_queue_time_seconds"),
@@ -2449,6 +2471,7 @@ class PrometheusLogger(CustomLogger):
         else:
             _metadata = {
                 "user_api_key_alias": getattr(_metadata_raw, "user_api_key_alias", None),
+                "user_api_key_user_email": getattr(_metadata_raw, "user_api_key_user_email", None),
                 "user_api_key_team_id": getattr(_metadata_raw, "user_api_key_team_id", None),
                 "user_api_key_team_alias": getattr(_metadata_raw, "user_api_key_team_alias", None),
                 "user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
@@ -2471,6 +2494,17 @@ class PrometheusLogger(CustomLogger):
                 return getattr(user_api_key_auth, "key_alias", None)
             return None
 
+        def _get_user_email() -> str | None:
+            from_metadata: Final = _metadata.get("user_api_key_user_email")
+            if from_metadata is not None:
+                return from_metadata
+            from_params: Final = _litellm_params_metadata.get("user_api_key_user_email")
+            if from_params is not None:
+                return from_params
+            if user_api_key_auth is not None:
+                return self._safe_get(user_api_key_auth, "user_email")
+            return None
+
         def _get_team_id() -> str | None:
             val = _metadata.get("user_api_key_team_id")
             if val is not None:
@@ -2506,6 +2540,7 @@ class PrometheusLogger(CustomLogger):
 
         return {
             "api_key_alias": _get_api_key_alias(),
+            "user_email": _get_user_email(),
             "team": _get_team_id(),
             "team_alias": _get_team_alias(),
             "hashed_api_key": _get_hashed_api_key(),
@@ -2563,6 +2598,7 @@ class PrometheusLogger(CustomLogger):
             _metadata: Final = standard_logging_payload.get("metadata", {}) or {}
             hashed_api_key: Final = fallback_values.get("hashed_api_key") or _metadata.get("user_api_key_hash")
             api_key_alias: Final = fallback_values.get("api_key_alias") or _metadata.get("user_api_key_alias")
+            user_email: Final = fallback_values.get("user_email")
             team: Final = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
             team_alias: Final = fallback_values.get("team_alias") or _metadata.get("user_api_key_team_alias")
             client_ip: Final = fallback_values.get("client_ip") or _metadata.get("requester_ip_address")
@@ -2603,6 +2639,7 @@ class PrometheusLogger(CustomLogger):
                 requested_model=label_requested_model,
                 hashed_api_key=hashed_api_key,
                 api_key_alias=api_key_alias,
+                user_email=user_email,
                 team=team,
                 team_alias=team_alias,
                 tags=standard_logging_payload.get("request_tags", []),
@@ -3539,7 +3576,9 @@ class PrometheusLogger(CustomLogger):
         except Exception as e:
             verbose_logger.exception("Error initializing user/team count metrics: %s", e)
 
-    async def _set_key_list_budget_metrics(self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]):
+    async def _set_key_list_budget_metrics(
+        self, keys: list[str | UserAPIKeyAuth | LiteLLM_DeletedVerificationToken]
+    ) -> None:
         """Helper function to set budget metrics for a list of keys"""
         for key in keys:
             if isinstance(key, UserAPIKeyAuth):

litellm/integrations/prompt_management_base.py

@@ -19,6 +19,19 @@ class PromptManagementClient(TypedDict):
     completed_messages: list[AllMessageValues] | None
 
 
+def resolve_prompt_manager_ignore_flags(
+    prompt_spec: PromptSpec | None,
+    ignore_prompt_manager_model: bool | None,
+    ignore_prompt_manager_optional_params: bool | None,
+) -> tuple[bool, bool]:
+    spec_params: Final = prompt_spec.litellm_params if prompt_spec is not None else None
+    return (
+        bool(ignore_prompt_manager_model) or bool(spec_params is not None and spec_params.ignore_prompt_manager_model),
+        bool(ignore_prompt_manager_optional_params)
+        or bool(spec_params is not None and spec_params.ignore_prompt_manager_optional_params),
+    )
+
+
 class PromptManagementBase(ABC):
     @property
     @abstractmethod
@@ -182,13 +195,18 @@ class PromptManagementBase(ABC):
             prompt_version=prompt_version,
         )
 
+        resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
+            prompt_spec=prompt_spec,
+            ignore_prompt_manager_model=ignore_prompt_manager_model,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+        )
         return self.post_compile_prompt_processing(
             prompt_template=prompt_template,
             messages=messages,
             non_default_params=non_default_params,
             model=model,
-            ignore_prompt_manager_model=ignore_prompt_manager_model,
-            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+            ignore_prompt_manager_model=resolved_ignore_model,
+            ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
         )
 
     async def async_get_chat_completion_prompt(
@@ -224,11 +242,16 @@ class PromptManagementBase(ABC):
             prompt_version=prompt_version,
         )
 
+        resolved_ignore_model, resolved_ignore_optional_params = resolve_prompt_manager_ignore_flags(
+            prompt_spec=prompt_spec,
+            ignore_prompt_manager_model=ignore_prompt_manager_model,
+            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+        )
         return self.post_compile_prompt_processing(
             prompt_template=prompt_template,
             messages=messages,
             non_default_params=non_default_params,
             model=model,
-            ignore_prompt_manager_model=ignore_prompt_manager_model,
-            ignore_prompt_manager_optional_params=ignore_prompt_manager_optional_params,
+            ignore_prompt_manager_model=resolved_ignore_model,
+            ignore_prompt_manager_optional_params=resolved_ignore_optional_params,
         )

litellm/integrations/s3_v2.py

@@ -11,6 +11,7 @@ import time
 from collections.abc import Mapping
 from datetime import datetime
 from typing import Final, cast
+from urllib.parse import quote
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -206,6 +207,23 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             params.get("s3_sse_kms_key_id") or s3_sse_kms_key_id,
         )
 
+    def _build_object_url(self, s3_object_key: str) -> str:
+        """
+        Build the exact URL that is both signed and sent, with the key percent-encoded once.
+
+        S3SigV4Auth signs the path verbatim while S3 canonicalizes the received path with reserved
+        characters encoded, so an unencoded `=`, `+`, `&`, `#`, `?`, `%` or space in the key makes
+        the two signatures disagree (403 SignatureDoesNotMatch).
+        """
+        encoded_key: Final = quote(s3_object_key, safe="/")
+        if self.s3_endpoint_url and self.s3_bucket_name:
+            if self.s3_use_virtual_hosted_style:
+                endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
+                protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
+                return f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{encoded_key}"
+            return f"{self.s3_endpoint_url}/{self.s3_bucket_name}/{encoded_key}"
+        return f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{encoded_key}"
+
     def _sse_headers(self) -> Mapping[str, str]:
         candidates: Final = {
             "x-amz-server-side-encryption": self.s3_server_side_encryption,
@@ -292,7 +310,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -316,18 +333,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             verbose_logger.debug("s3_v2 logger - uploading data to s3 - %s", batch_logging_element.s3_object_key)
             verbose_logger.debug("s3_v2 logger - s3_verify setting: %s", self.s3_verify)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -348,29 +354,19 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = await self.async_httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = await self.async_httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -478,7 +474,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             import base64
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
             from botocore.credentials import Credentials
@@ -493,18 +488,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 aws_region_name=self.s3_region_name,
             )
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{batch_logging_element.s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{batch_logging_element.s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + batch_logging_element.s3_object_key
+            url: Final = self._build_object_url(batch_logging_element.s3_object_key)
 
             # Convert JSON to string
             json_string: Final = safe_dumps(batch_logging_element.payload)
@@ -525,32 +509,22 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
                 "Cache-Control": "private, immutable, max-age=31536000, s-maxage=0",
                 **self._sse_headers(),
             }
-            req: Final = requests.Request("PUT", url, data=json_string, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                data=prepped.body,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="PUT", url=url, data=json_string, headers=headers)
             aws_region_name: Final = self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=self.s3_region_name)
             S3SigV4Auth(credentials, "s3", aws_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            # Use prepared URL so path segments match SigV4 canonical request (e.g. %20 for spaces).
-            request_url: Final = prepped.url or url
-
             httpx_client: Final = _get_httpx_client(
                 params=({"ssl_verify": self.s3_verify} if self.s3_verify is not None else None)
             )
             # Make the request with retry for transient S3 errors (500/503)
             max_retries: Final = 3
             for attempt in range(max_retries):
-                response = httpx_client.put(request_url, data=json_string, headers=signed_headers)
+                response = httpx_client.put(url, data=json_string, headers=signed_headers)
                 if response.status_code in (500, 503) and attempt < max_retries - 1:
                     wait_time = 2**attempt  # 1s, 2s
                     verbose_logger.warning(
@@ -582,7 +556,6 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         try:
             import hashlib
 
-            import requests
             from botocore.auth import S3SigV4Auth
             from botocore.awsrequest import AWSRequest
         except ImportError:
@@ -607,18 +580,7 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
 
             verbose_logger.debug("s3_v2 logger - downloading data from s3 - %s", s3_object_key)
 
-            # Prepare the URL
-            url = f"https://{self.s3_bucket_name}.s3.{self.s3_region_name}.amazonaws.com/{s3_object_key}"
-
-            if self.s3_endpoint_url and self.s3_bucket_name:
-                if self.s3_use_virtual_hosted_style:
-                    # Virtual-hosted-style: bucket.endpoint/key
-                    endpoint_host: Final = self.s3_endpoint_url.replace("https://", "").replace("http://", "")
-                    protocol: Final = "https://" if self.s3_endpoint_url.startswith("https://") else "http://"
-                    url = f"{protocol}{self.s3_bucket_name}.{endpoint_host}/{s3_object_key}"
-                else:
-                    # Path-style: endpoint/bucket/key
-                    url = self.s3_endpoint_url + "/" + self.s3_bucket_name + "/" + s3_object_key
+            url: Final = self._build_object_url(s3_object_key)
 
             # Prepare the request for GET operation
             # For GET requests, we need x-amz-content-sha256 with hash of empty string
@@ -626,22 +588,15 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
             headers: Final = {
                 "x-amz-content-sha256": empty_string_hash,
             }
-            req: Final = requests.Request("GET", url, headers=headers)
-            prepped: Final = req.prepare()
 
             # Sign the request
-            aws_request: Final = AWSRequest(
-                method=prepped.method,
-                url=prepped.url,
-                headers=prepped.headers,
-            )
+            aws_request: Final = AWSRequest(method="GET", url=url, headers=headers)
             S3SigV4Auth(credentials, "s3", self.s3_region_name).add_auth(aws_request)
 
             # Prepare the signed headers
             signed_headers: Final = dict(aws_request.headers.items())
 
-            request_url: Final = prepped.url or url
-            response: Final = await self.async_httpx_client.get(request_url, headers=signed_headers)
+            response: Final = await self.async_httpx_client.get(url, headers=signed_headers)
 
             if response.status_code != 200:
                 verbose_logger.exception("S3 object not found, saw response=", response.text)

litellm/integrations/shadow_eval_logger.py

@@ -452,6 +452,14 @@ async def _key_or_team_is_over_budget(metadata: Mapping[str, object]) -> bool:
     return False
 
 
+def _forwarded_team_id(metadata: Mapping[str, object]) -> str | None:
+    """The shadowed key's team, the identity the judge call already carries in its metadata
+    and the router already selects deployments with. Read here too so the arm choice, which
+    happens before the router sees the call, is made under the same team."""
+    team_id: Final = metadata.get("user_api_key_team_id")
+    return team_id if isinstance(team_id, str) and team_id else None
+
+
 def _routing_decision(metadata: Mapping[str, object]) -> Mapping[str, object]:
     """The routing decision a pre-routing strategy wrote to a call's metadata, empty when
     a plain model served it. Read off the sampled request for the control arm, and off the
@@ -915,6 +923,7 @@ class ShadowEvalLogger(CustomLogger):
                 self._router_provider(),
                 judge_model,
                 judge_messages,  # pyright: ignore[reportArgumentType]  # plain SDK message dicts
+                team_id=_forwarded_team_id(parent_metadata),
                 temperature=0,
                 max_tokens=JUDGE_MAX_OUTPUT_TOKENS,
                 response_format=PAIRWISE_JUDGE_RESPONSE_FORMAT,

litellm/integrations/websearch_interception/ARCHITECTURE.md

@@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
 
 ---
 
+## Loop Ceiling
+
+One intercepted request can chain several follow-up model calls, since the model often searches again after
+reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
+to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
+
+Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
+
+```yaml
+litellm_settings:
+  websearch_interception_params:
+    enabled_providers: ["bedrock"]
+    max_agentic_loops: 5
+```
+
+Or per deployment, which wins over the feature-level setting:
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5
+    litellm_params:
+      model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
+      max_agentic_loops: 5
+```
+
+Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
+carries it is ignored and one request can never drive an unbounded number of upstream model calls.
+
+Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
+from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
+not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
+validator down there would drop the deployment silently instead of refusing to start.
+
+When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
+the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
+The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
+The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
+buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
+
+Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
+treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
+rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
+client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
+already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
+rather than as a replacement.
+
+Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
+not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
+rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
+Both are tracked separately
+
+---
+
 ## Streaming Support
 
 WebSearch interception works transparently with both streaming and non-streaming requests.

litellm/integrations/websearch_interception/handler.py

@@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
 from litellm.integrations.websearch_interception.transformation import (
     WebSearchTransformation,
 )
+from litellm.litellm_core_utils.agentic_loop_settings import (
+    validated_max_agentic_loops,
+)
 from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
@@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         self,
         enabled_providers: list[LlmProviders | str] | None = None,
         search_tool_name: str | None = None,
+        max_agentic_loops: int | None = None,
     ):
         """
         Args:
@@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
                               Default: None (all providers enabled)
             search_tool_name: Name of search tool configured in router's search_tools.
                              If None, will attempt to use first available search tool.
+            max_agentic_loops: How many follow-up model calls one intercepted request
+                              may chain before the loop is refused and the turn ends.
+                              If None, LiteLLM's default of 3 applies.
         """
         super().__init__()
         # Convert enum values to strings for comparison
@@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
         else:
             self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
         self.search_tool_name = search_tool_name
+        self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
         self._request_has_websearch = False  # Track if current request has web search
 
+    @staticmethod
+    def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
+        """
+        Reject loop ceilings the agentic loop cannot honor, at config load time.
+        """
+        return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
+
     async def try_short_circuit_search(
         self,
         model: str,
@@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
                   websearch_interception_params:
                     enabled_providers: ["bedrock"]
                     search_tool_name: "my-perplexity-search"
+                    max_agentic_loops: 5
 
             Usage:
                 config = litellm_settings.get("websearch_interception_params", {})
@@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         # Extract parameters from config
         enabled_providers_str: Final = config.get("enabled_providers", None)
         search_tool_name: Final = config.get("search_tool_name", None)
+        max_agentic_loops: Final = config.get("max_agentic_loops", None)
 
         # Convert string provider names to LlmProviders enum values
         enabled_providers: list[LlmProviders | str] | None = None
@@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         return cls(
             enabled_providers=enabled_providers,
             search_tool_name=search_tool_name,
+            max_agentic_loops=max_agentic_loops,
         )
 
     @staticmethod
@@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
 
         verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
 
+        deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
+        if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
+            kwargs["max_agentic_loops"] = self.max_agentic_loops  # rebind-ok: this hook returns the kwargs it edits
+
         # If the client sent an Anthropic-native web_search_* tool, mark the
         # request so the agentic loop emits native web_search_tool_result
         # blocks in the final response (for citations panels, etc.). The flag

litellm/interactions/background_cost_polling.py

@@ -0,0 +1,313 @@
+"""
+Cost tracking for background interactions.
+
+A create request with ``background=true`` returns ``in_progress`` with no
+usage block, and GET polls are deliberately never billed (billing them would
+double-charge every poll; the GET response also does not echo ``background``,
+so a poll cannot be told apart from a re-fetch of an already-billed
+interaction). The create call is therefore the only place that can own
+billing: it schedules a poll task that fetches the interaction until it
+reaches a terminal status and logs the final usage as a single success event
+attributed to the original request.
+
+``requires_action`` is terminal for the interaction it names. The API has no
+operation that resumes one: a caller answers a tool request by creating a new
+interaction whose ``previous_interaction_id`` points at it, and that new
+interaction bills itself. The paused interaction keeps the tokens it already
+spent producing the tool request, so it is billed and settled where it stops
+rather than polled until the timeout, which would both lose that usage and
+hold its budget reservation open for the whole timeout window.
+
+Deleting an interaction makes every subsequent poll fail, which would let a
+caller retrieve the completed output themselves and then delete it before the
+poll task settles, leaving the work unbilled and the budget reservation
+refunded at the poll timeout. ``adelete`` therefore settles any pending poll
+for the interaction before dispatching the delete: it fetches the current
+state with the create's credentials, bills it if it is terminal with usage,
+and releases the reservation otherwise. A settlement gate on the create's
+logging object makes the poll task and the delete path mutually exclusive, so
+the interaction is billed exactly once no matter who settles first.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable, Iterator, Mapping
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+from litellm._logging import verbose_logger
+from litellm.constants import (
+    BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
+    BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
+)
+from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
+from litellm.types.interactions import InteractionsAPIResponse
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+_TERMINAL_STATUSES: Final = frozenset(
+    {"completed", "failed", "cancelled", "incomplete", "budget_exceeded", "requires_action"}
+)
+
+_POLLABLE_STATUSES: Final = frozenset({"in_progress", "queued"})
+
+_STATUSES_THAT_PRODUCED_OUTPUT: Final = frozenset({"completed", "requires_action"})
+
+
+@dataclass(frozen=True, slots=True)
+class BackgroundInteractionPollContext:
+    interaction_id: str
+    custom_llm_provider: str
+    logging_obj: "LiteLLMLoggingObj"
+    api_key: str | None = None
+    api_base: str | None = None
+    initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
+    max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
+    timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
+
+
+FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Awaitable[InteractionsAPIResponse]]
+
+
+async def _fetch_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
+    from litellm.interactions import aget
+
+    return await aget(
+        interaction_id=context.interaction_id,
+        custom_llm_provider=context.custom_llm_provider,
+        api_key=context.api_key,
+        api_base=context.api_base,
+        **{
+            "no-log": True
+        },  # mutable-ok: "no-log" is not a valid identifier, so it can only be passed through a mapping
+    )
+
+
+def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
+    elapsed = 0.0
+    interval = initial
+    while interval > 0 and elapsed + interval <= timeout:
+        yield interval
+        elapsed += interval
+        interval = min(interval * 2, maximum)
+
+
+_SETTLED_KEY = "background_interaction_settled"
+
+
+def _is_settled(logging_obj: "LiteLLMLoggingObj") -> bool:
+    return logging_obj.model_call_details.get(_SETTLED_KEY) is True
+
+
+def _claim_settlement(logging_obj: "LiteLLMLoggingObj") -> bool:
+    """
+    Exactly-once gate between the poll task and the delete-time settlement:
+    both run on the same event loop and neither awaits between reading and
+    setting the flag, so whichever claims first owns billing or release.
+    """
+    if _is_settled(logging_obj):
+        return False
+    logging_obj.model_call_details[_SETTLED_KEY] = True  # rebind-ok: both settlers must see the same settlement flag
+    return True
+
+
+async def poll_and_log_background_interaction_cost(
+    context: BackgroundInteractionPollContext,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    last_seen_status: str | None = None
+    for interval in _poll_intervals(
+        initial=context.initial_interval_seconds,
+        maximum=context.max_interval_seconds,
+        timeout=context.timeout_seconds,
+    ):
+        await asyncio.sleep(interval)
+        if _is_settled(context.logging_obj):
+            return
+        try:
+            response = await fetch_interaction(context)
+        except Exception as e:  # noqa: BLE001  # any fetch error must not kill the billing poll loop
+            verbose_logger.debug(
+                "Background interaction cost poll for %s failed, will retry: %s",
+                context.interaction_id,
+                e,
+            )
+            continue
+        last_seen_status = response.status
+        if response.status not in _TERMINAL_STATUSES:
+            continue
+        if not _claim_settlement(context.logging_obj):
+            return
+        if response.usage is not None:
+            await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        else:
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if last_seen_status is not None and last_seen_status not in _POLLABLE_STATUSES:
+        verbose_logger.error(
+            "Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
+            "the pollable nor the terminal set, so this proxy never learned how to settle it and its usage "
+            "will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+            last_seen_status,
+        )
+    else:
+        verbose_logger.warning(
+            "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
+            context.interaction_id,
+            context.timeout_seconds,
+        )
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)
+
+
+async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
+    """
+    The proxy keeps the pre-call budget reservation open for an in-progress
+    background interaction so concurrent creates cannot stack past the budget.
+    The completion success event reconciles it to the actual cost; when the
+    interaction terminates without billable usage (or polling gives up, or it
+    is deleted before settling), no such event fires, so whoever claims the
+    settlement must release the reservation here or the spend counters stay
+    pinned at the estimated cost.
+    """
+    metadata = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
+    budget_reservation = metadata.get("user_api_key_budget_reservation")
+    if not isinstance(budget_reservation, dict):
+        return
+
+    from litellm.proxy.spend_tracking.budget_reservation import release_budget_reservation
+
+    try:
+        await release_budget_reservation(budget_reservation=budget_reservation)
+    except Exception:  # noqa: BLE001  # a failed release must not crash the poll task; counters expire via TTL
+        verbose_logger.exception("Failed to release budget reservation for an unbilled background interaction")
+
+
+async def _bill_settled_interaction(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> None:
+    """
+    Claiming the settlement makes the claimer solely responsible for the
+    reservation, and no one retries a claim that is already set. A billing
+    failure here must therefore release the reservation on its way out, or it
+    stays pinned at the estimated cost until the whole poll times out.
+    """
+    try:
+        await logging_obj.async_log_background_interaction_completion(result=response)
+    except Exception:
+        await _release_open_budget_reservation(logging_obj=logging_obj)
+        raise
+
+
+def is_pollable_background_interaction(response: InteractionsAPIResponse) -> bool:
+    """
+    The single gate deciding whether a create's response gets a poll task.
+    The proxy's success callback defers releasing the budget reservation for
+    exactly these responses, on the promise that a poll task will settle them,
+    so a response one site accepts and the other refuses strands its
+    reservation on the spend counters with nothing left to reconcile it.
+
+    ``queued`` belongs here alongside ``in_progress``. It is the API's
+    not-started-yet state, so it reaches a terminal status the same way and
+    needs polling for the same reason: nothing else in the proxy ever bills a
+    create that came back without usage, so a status missing from both this
+    set and ``_TERMINAL_STATUSES`` is billed nowhere and alerts nobody.
+    """
+    return response.status in _POLLABLE_STATUSES and bool(response.id)
+
+
+def missing_usage_is_expected(response: InteractionsAPIResponse) -> bool:
+    """
+    Whether a response arriving with no usage block is a normal outcome rather
+    than lost billing data. An interaction that is still running, or that
+    stopped at ``failed``, ``cancelled``, ``incomplete`` or ``budget_exceeded``,
+    has nothing to charge for and should not raise a cost-tracking alarm.
+
+    ``completed`` and ``requires_action`` both mean the model produced output,
+    so a usage block is always expected with them. If one arrives without it
+    the charge for real work has been lost, which is precisely what the
+    proxy's cost-tracking alert exists to surface.
+    """
+    return response.status not in _STATUSES_THAT_PRODUCED_OUTPUT
+
+
+@dataclass(frozen=True, slots=True)
+class _ActiveBackgroundPoll:
+    task: "asyncio.Task[None]"
+    context: BackgroundInteractionPollContext
+
+
+_ACTIVE_POLLS: dict[str, _ActiveBackgroundPoll] = {}  # mutable-ok: asyncio needs strong refs to running poll tasks
+
+
+def _discard_poll(interaction_id: str, task: "asyncio.Task[None]") -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is not None and entry.task is task:
+        del _ACTIVE_POLLS[interaction_id]
+
+
+def maybe_schedule_background_interaction_cost_polling(
+    response: object,
+    create_kwargs: Mapping[str, object],
+    custom_llm_provider: str,
+) -> "asyncio.Task[None] | None":
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+    if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
+        return None
+    if not isinstance(response, InteractionsAPIResponse):
+        return None
+    if not is_pollable_background_interaction(response):
+        return None
+    logging_obj = create_kwargs.get("litellm_logging_obj")
+    if not isinstance(logging_obj, Logging):
+        return None
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        return None
+    api_key = create_kwargs.get("api_key")
+    api_base = create_kwargs.get("api_base")
+    context = BackgroundInteractionPollContext(
+        interaction_id=response.id,
+        custom_llm_provider=custom_llm_provider,
+        logging_obj=logging_obj,
+        api_key=api_key if isinstance(api_key, str) else None,
+        api_base=api_base if isinstance(api_base, str) else None,
+    )
+    task = asyncio.create_task(poll_and_log_background_interaction_cost(context))
+    _ACTIVE_POLLS[context.interaction_id] = _ActiveBackgroundPoll(task=task, context=context)
+    task.add_done_callback(
+        lambda finished, interaction_id=context.interaction_id: _discard_poll(interaction_id, finished)
+    )
+    return task
+
+
+async def maybe_settle_background_interaction_before_delete(
+    interaction_id: str,
+    fetch_interaction: FetchInteraction = _fetch_interaction,
+) -> None:
+    entry = _ACTIVE_POLLS.get(interaction_id)
+    if entry is None:
+        return
+    context = entry.context
+    try:
+        response = await fetch_interaction(context)
+    except Exception as e:  # noqa: BLE001  # unfetchable pre-delete state settles by releasing the reservation
+        verbose_logger.debug(
+            "Could not fetch background interaction %s before delete, releasing its reservation: %s",
+            interaction_id,
+            e,
+        )
+        if _claim_settlement(context.logging_obj):
+            await _release_open_budget_reservation(logging_obj=context.logging_obj)
+        return
+    if not _claim_settlement(context.logging_obj):
+        return
+    if response.status in _TERMINAL_STATUSES and response.usage is not None:
+        await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+        return
+    await _release_open_budget_reservation(logging_obj=context.logging_obj)