Merge pull request #37913 from BerriAI/litellm_internal_staging

chore(ci): promote internal staging to main

.github/CODEOWNERS

@@ -1,5 +1,7 @@
-/ui/ @yuneng-jiang @ryan-crabbe-berri
-/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/ @yuneng-berri @ryan-crabbe-berri
+/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
+/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri
+/.github/CODEOWNERS @yuneng-berri

.github/actions/cache-cargo-build/action.yml

@@ -0,0 +1,31 @@
+name: "Cache the Rust build"
+description: >-
+  Cache the Cargo registry and target directory the root package's build needs,
+  so only the first job on a given Cargo.lock compiles the bridge from scratch.
+
+  litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
+  in release mode before it can produce a wheel. `uv sync` therefore pays a full
+  build in every job that installs the workspace: measured at 2m40s per unit shard
+  on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
+  it, because the uv cache holds wheels uv downloads rather than wheels it builds,
+  and a path dependency whose source moves every commit could never hit that cache
+  anyway. Cargo rebuilds only what changed when its target directory survives, so a
+  warm job pays for the bridge crate alone.
+
+  The key namespace is separate from test-rust.yml's. Both cache the same directory,
+  but that workflow fills it with debug and clippy artifacts, which a release build
+  cannot reuse, and a shared key would let whichever ran first deny the other a save.
+
+runs:
+  using: composite
+  steps:
+    - name: Restore the Cargo registry and target directory
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          litellm-rust/target
+        key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-cargo-release-

.github/ci-coverage-allowlist.yml

@@ -48,16 +48,6 @@ test_paths:
       choice it informed is settled
     paths:
       - tests/code_coverage_tests/test_aio_http_image_conversion.py
-  - reason: >-
-      The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
-      other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
-      their bodies into the live file of the same name. This one cannot follow either route yet:
-      its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
-      counterpart while 25 assertions fail against today's code, so what survives that rewrite
-      is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
-      behaviours still hold
-    paths:
-      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
   - reason: >-
       No job invokes this suite and its files mix pure transformation tests with ones driving live
       vendor vector stores, so assigning them needs a per-file decision

.github/workflows/_test-unit-base.yml

@@ -27,7 +27,7 @@ on:
         default: 20
       job-timeout-minutes:
         description: >-
-          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
           the per-step ceilings on the setup steps below, and 5 for the runner
           overhead the job clock charges but no step owns (job init, step
           transitions, post-job cleanup). That headroom is what makes the test
@@ -36,7 +36,7 @@ on:
           arithmetic, so the sum is passed in rather than computed.
         required: false
         type: number
-        default: 55
+        default: 60
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -103,6 +103,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8

.github/workflows/check-ui-api-types.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install backend dependencies
         if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

.github/workflows/image-scan.yml

@@ -17,6 +17,8 @@ on:
       - backend/Dockerfile
       - backend/main.py
       - docker/component_entrypoint.sh
+      - docker/entrypoint.sh
+      - litellm/proxy/prisma_migration.py
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock

.github/workflows/mutation-test.yml

@@ -53,6 +53,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml

.github/workflows/publish-basedpyright-base-counts.yml

@@ -43,6 +43,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Cache Prisma binaries
         uses: ./.github/actions/cache-prisma-binaries
 

.github/workflows/test-code-quality.yml

@@ -56,6 +56,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: uv sync --frozen --all-groups --all-extras
 

.github/workflows/test-linting.yml

@@ -78,6 +78,10 @@ jobs:
         run: |
           uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-mcp.yml

@@ -47,6 +47,10 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-terraform-provider.yml

@@ -88,6 +88,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

.github/workflows/test-unit-documentation.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-unit.yml

@@ -55,7 +55,7 @@ jobs:
             workers: 2
             reruns: 1
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
@@ -67,7 +67,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: integrations
             artifact-name: integrations
@@ -75,7 +75,7 @@ jobs:
             workers: 2
             reruns: 3
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
@@ -83,7 +83,7 @@ jobs:
             workers: 1
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
@@ -91,7 +91,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: misc
             artifact-name: misc
@@ -122,7 +122,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-auth
             artifact-name: proxy-auth
@@ -134,7 +134,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
@@ -171,7 +171,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-server
             artifact-name: proxy-server
@@ -179,7 +179,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 60
-            job-timeout-minutes: 95
+            job-timeout-minutes: 100
 
           - shard: proxy-infra
             artifact-name: proxy-infra
@@ -198,7 +198,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
@@ -209,7 +209,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}

.github/workflows/weekly_load_anomaly.yml

@@ -47,6 +47,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

basedpyright-code-budget.json

@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1823
+    "limit": 1822
   },
   "reportRedeclaration": {
     "limit": 8

ci_cd/generate_model_prices_schema.py

@@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
         "uses_embed_content",
         "use_openai_responses_path",
         "bedrock_converse_supports_strict_tools",
+        "thinking_always_on",
     }
 )
 

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.

enterprise/enterprise_hooks/banned_keywords.py

@@ -21,6 +21,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BannedKeywords(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self):
         banned_keywords_list = litellm.banned_keywords_list

enterprise/enterprise_hooks/blocked_user_list.py

@@ -18,6 +18,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BlockedUserList(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self, prisma_client: Optional[PrismaClient]):
         self.prisma_client = prisma_client

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -966,6 +966,16 @@ class CheckBatchCost:
                     )
 
             elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                from litellm.proxy.openai_files_endpoints.common_utils import (
+                    _completed_batch_safe_to_retire,
+                )
+
+                if response.status in ("completed", "complete") and not _completed_batch_safe_to_retire(response):
+                    verbose_proxy_logger.info(
+                        f"CheckBatchCost: batch {batch_id} is completed but its output file id "
+                        f"has not appeared yet; leaving job {job.id} for the next poll cycle"
+                    )
+                    continue
                 await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -45,6 +45,8 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
+    FILE_LIST_CONTINUATION_CHUNK_SIZE,
+    MAX_FILE_LIST_LIMIT,
     _is_base64_encoded_unified_file_id,
     apply_unified_file_ids,
     ensure_batch_response_managed_file_ids,
@@ -54,6 +56,8 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     map_raw_file_ids_to_unified,
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
+    validate_file_list_limit,
+    validate_file_list_purpose,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
     request_tags_from_metadata,
@@ -63,9 +67,9 @@ from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccess
     AsyncCursorPage,
     ChatCompletionFileObject,
     CreateFileRequest,
+    FileListPage,
     FileObject,
     OpenAIFileObject,
-    OpenAIFilesPurpose,
     ResponsesAPIResponse,
 )
 from litellm.types.utils import (
@@ -144,7 +148,14 @@ class _ManagedFileRow(Protocol):
 class _ManagedFileTableActions(Protocol):
     async def find_first(self, where: Mapping[str, object]) -> Optional[_ManagedFileRow]: ...
 
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[_ManagedFileRow]: ...
+    async def find_many(
+        self,
+        where: Mapping[str, object],
+        take: int = ...,
+        order: Union[Mapping[str, str], Sequence[Mapping[str, str]]] = ...,
+        cursor: Mapping[str, str] = ...,
+        skip: int = ...,
+    ) -> Sequence[_ManagedFileRow]: ...
 
     async def upsert(self, where: Mapping[str, str], data: Mapping[str, Mapping[str, object]]) -> _ManagedFileRow: ...
 
@@ -1365,12 +1376,76 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
 
     async def afile_list(
         self,
-        purpose: Optional[OpenAIFilesPurpose],
+        purpose: Optional[str],
         litellm_parent_otel_span: Optional[Span],
+        user_api_key_dict: UserAPIKeyAuth,
+        limit: Optional[int] = None,
+        after: Optional[str] = None,
         **data: Dict,
-    ) -> List[OpenAIFileObject]:
-        """Handled in files_endpoints.py"""
-        return []
+    ) -> FileListPage:
+        """List the managed files the caller owns, newest first.
+
+        Pagination is keyset based on ``unified_file_id`` so a key that owns
+        every file on the proxy still reads one bounded page at a time.
+        ``purpose`` is applied after parsing, because the managed file table
+        keeps it inside the ``file_object`` blob instead of a column, and rows
+        whose blob will not parse drop out there too, so a chunk of rows can
+        yield fewer matches than the page holds. Successive chunks are read
+        until the page is full or the caller's rows run out, which keeps
+        ``data`` non-empty while matches remain and its last id usable as the
+        next cursor. A first chunk that fills the page costs one query; once a
+        scan has to continue past it, the chunk widens to
+        ``FILE_LIST_CONTINUATION_CHUNK_SIZE``, so the walk costs one query per
+        that many rows instead of one per page. That bound is per query, not
+        per request: the work is still linear in the rows the caller owns, and
+        a filter matching nothing reads every one of them, with no index
+        covering either the owner filter or the sort.
+        """
+        validate_file_list_limit(limit)
+        validate_file_list_purpose(purpose)
+
+        owner_filter: Final = build_owner_filter(user_api_key_dict)
+        if owner_filter is None:
+            return FileListPage(**build_list_page([]))
+
+        if after:
+            cursor_row = await _managed_file_table(self.prisma_client).find_first(
+                where={**owner_filter, "unified_file_id": after}
+            )
+            if cursor_row is None:
+                raise ProxyException(
+                    message=f"Invalid 'after' cursor: no file found with id '{after}'.",
+                    type="invalid_request_error",
+                    param="after",
+                    code=400,
+                    openai_code="invalid_value",
+                )
+
+        page_size: Final = min(limit or MAX_FILE_LIST_LIMIT, MAX_FILE_LIST_LIMIT)
+        matches: Final[List[OpenAIFileObject]] = []
+        cursor_id = after
+        chunk_size = page_size + 1
+
+        while len(matches) <= page_size:
+            cursor_args: _CursorPageArgs = {"cursor": {"unified_file_id": cursor_id}, "skip": 1} if cursor_id else {}
+            chunk = await _managed_file_table(self.prisma_client).find_many(
+                where=owner_filter,
+                take=chunk_size,
+                order=[{"created_at": "desc"}, {"unified_file_id": "desc"}],
+                **cursor_args,
+            )
+            matches.extend(
+                parsed_file_object.model_copy(update={"id": row.unified_file_id})
+                for row in chunk
+                if (parsed_file_object := _parse_managed_file_object(row.file_object, row.unified_file_id)) is not None
+                and (purpose is None or parsed_file_object.purpose == purpose)
+            )
+            if len(chunk) < chunk_size:
+                break
+            cursor_id = chunk[-1].unified_file_id
+            chunk_size = max(chunk_size, FILE_LIST_CONTINUATION_CHUNK_SIZE)
+
+        return FileListPage(**build_list_page(matches[:page_size], has_more=len(matches) > page_size))
 
     def _is_batch_polling_enabled(self) -> bool:
         """

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.58"
+version = "0.1.59"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.58"
+version = "0.1.59"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:a31344ab2cb8618db84f535eec56f76f6178b142cb92cb2e48676cc2dcebea72
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql

@@ -1,4 +0,0 @@
-UPDATE "LiteLLM_SpendLogs"
-SET "created_at" = "endTime",
-    "updated_at" = "endTime"
-WHERE "created_at" > "endTime" + interval '1 hour';

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.88"
+version = "0.4.89"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.88"
+version = "0.4.89"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/_logging.py

@@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str:
     return _redact_string(value)
 
 
+def _substituted_color_message(record: logging.LogRecord) -> str | None:
+    """Render a record's ``color_message`` against its args, or None if absent.
+
+    uvicorn's colorized formatter re-renders `color_message` against
+    record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead
+    of using the already-formatted record.msg, so it has to be substituted
+    before args are cleared or it is later formatted with no args and prints
+    the raw "%s://%s:%d" placeholders instead of the URL.
+    """
+    color_message: Final = record.__dict__.get("color_message")
+    if not isinstance(color_message, str) or not record.args:
+        return None
+    try:
+        return color_message % record.args
+    except TypeError:
+        return color_message
+
+
 class SecretRedactionFilter(logging.Filter):
     """Scrubs known secret/credential patterns from log records."""
 
@@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter):
         if not _ENABLE_SECRET_REDACTION:
             return True
 
+        # Runs before args are cleared, and before the extra-field loop below
+        # that redacts the substituted result.
+        substituted_color_message: Final = _substituted_color_message(record)
+        if substituted_color_message is not None:
+            record.color_message = substituted_color_message  # rebind-ok: a Filter scrubs records in place
+
         try:
             record.msg = _redact_string(record.getMessage())
             record.args = None

litellm/_redis.py

@@ -665,8 +665,16 @@ def get_redis_async_client(
         cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
         cluster_kwargs.setdefault("socket_keepalive", True)
 
+        # A single node's client-side timeout must reset only that node's connections,
+        # not tear down the whole cluster client for every concurrent caller.
+        from litellm.caching.redis_cluster_node_isolation import (
+            get_litellm_async_redis_cluster_class,
+        )
+
+        async_redis_cluster_class: Final = get_litellm_async_redis_cluster_class()
+
         # Create async RedisCluster with IAM token as password if available
-        cluster_client: Final = async_redis.RedisCluster(
+        cluster_client: Final = async_redis_cluster_class(
             startup_nodes=new_startup_nodes,
             **cluster_kwargs,
         )

litellm/batches/batch_utils.py

@@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage(
     )
 
 
+def _provider_output_file_id(output_file_id: str) -> str:
+    """
+    Resolve the file id the provider actually knows: unified ids yield their embedded
+    llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through.
+    """
+    from litellm.proxy.openai_files_endpoints.common_utils import (
+        _is_base64_encoded_unified_file_id,
+        get_original_file_id,
+    )
+
+    unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id)
+    if not unified_file_id:
+        return get_original_file_id(output_file_id)
+    try:
+        extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
+    except (IndexError, AttributeError) as e:
+        verbose_logger.error(
+            "Failed to extract LLM output file ID from unified file ID: %s, error: %s",
+            output_file_id,
+            e,
+        )
+        return output_file_id
+    verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted)
+    return extracted
+
+
 async def _fetch_batch_output_file_content(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content(
                        Required for Azure and other providers that need authentication
     """
     from litellm.files.main import afile_content
-    from litellm.proxy.openai_files_endpoints.common_utils import (
-        _is_base64_encoded_unified_file_id,
-    )
 
     if batch.output_file_id is None:
         raise ValueError("Output file id is None cannot retrieve file content")
 
-    file_id = batch.output_file_id
-    is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id)
-    if is_base64_unified_file_id:
-        try:
-            file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
-            verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id)
-        except (IndexError, AttributeError) as e:
-            verbose_logger.error(
-                "Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e
-            )
+    file_id: Final = _provider_output_file_id(batch.output_file_id)
 
     # Build kwargs for afile_content with credentials from litellm_params
     file_content_kwargs: Final = {

litellm/caching/redis_cluster_node_isolation.py

@@ -0,0 +1,173 @@
+"""Bounds the blast radius of a single node's transient connection error on the async
+Redis Cluster client.
+
+redis-py's ``RedisCluster._execute_command`` responds to a ``ConnectionError`` or
+``TimeoutError`` on ANY one node by tearing down every node's connections and flipping
+the client into "needs reinitialization", which forces every other concurrent caller
+sharing this client through one reinit lock until the whole cluster topology is
+re-walked. Under real proxy load, a client-side socket timeout on a single node is a
+routine event (the event loop was too busy to read the response before ``socket_timeout``
+elapsed) and does not mean the cluster's topology moved, so treating it as a full-cluster
+event turns one slow node into a proxy-wide latency spike while Redis itself stays
+healthy -- confirmed live: pausing one of three local cluster nodes made every concurrent
+command against the other two, untouched nodes stall for the full pause duration too.
+
+``get_litellm_async_redis_cluster_class`` returns a ``RedisCluster`` subclass that resets
+only the node that actually failed (mirroring what a plain, non-cluster Redis client
+already does when one of its pooled connections errors), leaving every other node's
+connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
+retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
+topology changed.
+"""
+
+import asyncio
+from typing import TYPE_CHECKING, Final, Protocol
+
+from litellm._logging import verbose_logger
+
+if TYPE_CHECKING:
+    from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
+
+
+class _ClusterNodeAttrs(Protocol):
+    """The subset of ``redis.asyncio.cluster.ClusterNode`` this override reads. redis-py
+    ships no resolvable stub for these members under the repo's current types-redis pin,
+    so a plain attribute access resolves every downstream use to ``Unknown`` under strict
+    mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
+    own logic fully typed without a banned ``typing.cast``."""
+
+    async def execute_command(
+        self,
+        *args: object,
+        **kwargs: object,  # kwargs-ok: mirrors redis-py's own ClusterNode.execute_command signature, a raw command dispatch with no fixed keyword contract
+    ) -> object: ...
+    async def disconnect(self) -> None: ...
+
+
+class _NodesManagerAttrs(Protocol):
+    _moved_exception: object
+
+    def get_node_from_slot(
+        self, slot: int, read_from_replicas: bool, load_balancing_strategy: object
+    ) -> _ClusterNodeAttrs: ...
+
+
+class _ClusterAttrs(Protocol):
+    RedisClusterRequestTTL: int
+    reinitialize_counter: int
+    reinitialize_steps: int
+    read_from_replicas: bool
+    load_balancing_strategy: object
+    nodes_manager: _NodesManagerAttrs
+
+    def get_node(self, node_name: str) -> _ClusterNodeAttrs: ...
+    async def _determine_slot(self, *args: object) -> int: ...
+    async def aclose(self) -> None: ...
+
+
+#: redis-py versions this override's copied ``_execute_command`` body has been verified
+#: against. A version outside this set may have changed the method's structure in a way
+#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
+#: construction logs a loud warning rather than silently trusting an unverified copy.
+_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
+
+
+def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
+    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+
+    Imported lazily because this module is reachable from a base ``import litellm`` while
+    redis is not a base dependency. Cheap to call repeatedly: the underlying redis
+    submodules are cached in ``sys.modules`` after the first import.
+    """
+    import redis
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.cluster import get_node_name
+    from redis.commands import READ_COMMANDS
+    from redis.exceptions import (
+        AskError,
+        BusyLoadingError,
+        ClusterDownError,
+        ClusterError,
+        MaxConnectionsError,
+        MovedError,
+        SlotNotCoveredError,
+        TryAgainError,
+    )
+    from redis.exceptions import ConnectionError as _RedisConnectionError
+    from redis.exceptions import TimeoutError as _RedisTimeoutError
+
+    if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
+        verbose_logger.warning(
+            "redis-py %s is not in the set this cluster-teardown-storm fix was verified "
+            "against (%s). The per-node-isolation override may not match the installed library's "
+            "real _execute_command behavior.",
+            redis.__version__,
+            sorted(_VERIFIED_REDIS_VERSIONS),
+        )
+
+    class LiteLLMAsyncRedisCluster(
+        _BaseAsyncRedisCluster  # pyright: ignore[reportUntypedBaseClass]  # same stale-stub gap as the import above; the base class itself is unresolvable, not this subclass's own code
+    ):
+        async def _execute_command(
+            self,
+            target_node: _ClusterNodeAttrs,
+            *args: object,
+            **kwargs: object,  # kwargs-ok: overrides redis-py's own **kwargs signature; the keyword contract is defined by the Redis command being dispatched, not by this method
+        ) -> object:
+            cluster: _ClusterAttrs = self
+            node = target_node
+
+            asking = moved = False
+            redirect_addr: str | None = None
+            ttl = cluster.RedisClusterRequestTTL
+
+            while ttl > 0:
+                ttl -= 1
+                try:
+                    if asking:
+                        assert redirect_addr is not None
+                        node = cluster.get_node(node_name=redirect_addr)
+                        await node.execute_command("ASKING")
+                        asking = False
+                    elif moved:
+                        slot = await cluster._determine_slot(*args)  # pyright: ignore[reportPrivateUsage]  # mirrors upstream's own un-overridden branch, which makes this identical private call from the same subclass
+                        node = cluster.nodes_manager.get_node_from_slot(
+                            slot,
+                            cluster.read_from_replicas and args[0] in READ_COMMANDS,
+                            (cluster.load_balancing_strategy if args[0] in READ_COMMANDS else None),
+                        )
+                        moved = False
+
+                    return await node.execute_command(*args, **kwargs)
+                except (BusyLoadingError, MaxConnectionsError):
+                    raise
+                except (_RedisConnectionError, _RedisTimeoutError):
+                    # Reset only the node that actually failed instead of the upstream
+                    # default (`await self.aclose()`, a full-cluster teardown that forces
+                    # every other concurrent caller through the shared reinit lock).
+                    await node.disconnect()
+                    raise
+                except (ClusterDownError, SlotNotCoveredError):
+                    await cluster.aclose()
+                    await asyncio.sleep(0.25)
+                    raise
+                except MovedError as e:
+                    cluster.reinitialize_counter += 1
+                    if cluster.reinitialize_steps and cluster.reinitialize_counter % cluster.reinitialize_steps == 0:
+                        await cluster.aclose()
+                        cluster.reinitialize_counter = 0
+                    else:
+                        cluster.nodes_manager._moved_exception = e  # pyright: ignore[reportPrivateUsage]  # mirrors upstream's own un-overridden branch; redis-py exposes no public setter for this
+                    moved = True
+                except AskError as e:
+                    redirect_addr = get_node_name(host=e.host, port=e.port)
+                    asking = True
+                except TryAgainError:
+                    if ttl < cluster.RedisClusterRequestTTL / 2:
+                        await asyncio.sleep(0.05)
+
+            raise ClusterError("TTL exhausted.")
+
+    return LiteLLMAsyncRedisCluster

litellm/constants.py

@@ -783,6 +783,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://pinstripes.io/v1",
     "https://api.meta.ai/v1",
     "https://api.cognition.ai/v1",
+    "https://api.scx.ai/v1",
 ]
 
 
@@ -851,6 +852,7 @@ openai_compatible_providers: Final[list] = [
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
+    "scx-ai",
 ]
 openai_text_completion_compatible_providers: Final[list] = [  # providers that support `/v1/completions`
     "together_ai",
@@ -1354,6 +1356,8 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
 LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
+AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
+ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@@ -1534,6 +1538,7 @@ TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
+SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))

litellm/integrations/custom_logger.py

@@ -60,6 +60,25 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+
+    enforces_request_content: bool = False
+    """
+    Whether this hook's ``async_pre_call_hook`` judges the request payload itself.
+
+    False for the accounting hooks, which count a request rather than read it: rate limits,
+    parallel slots, budgets, cache lookups. Those must run once per request and never once per
+    record of a batch upload, which would charge a caller once for every line of their file.
+
+    Set it to True on a hook that inspects or rejects content, so that scanning a payload which
+    is not itself a request, such as one record of a batch input file, still reaches it. A
+    ``CustomGuardrail`` does not need it; guardrails are dispatched by their own branch.
+
+    Judging content is necessary but not sufficient. A hook that also rewrites the payload for
+    routing, as the managed-files and managed-vector-store hooks do, stays False: a per-record
+    rewrite would read as a redaction and ship embedded in the record. Only the leaf class is
+    consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
+    """
+
     def __init__(
         self,
         turn_off_message_logging: bool = False,

litellm/integrations/otel/__init__.py

@@ -49,6 +49,7 @@ from litellm.integrations.otel.model.semconv import (
     Error,
     GenAI,
     GenAIOperation,
+    GenAIOutputType,
     GenAIProvider,
     JsonRpc,
     LiteLLM,
@@ -60,6 +61,7 @@ from litellm.integrations.otel.model.semconv import (
     RpcSystem,
     Server,
     resolve_operation,
+    resolve_output_type,
     resolve_provider,
 )
 from litellm.integrations.otel.model.spans import (
@@ -84,6 +86,7 @@ __all__ = [
     "Error",
     "GenAI",
     "GenAIOperation",
+    "GenAIOutputType",
     "GenAIProvider",
     "GuardrailSpanData",
     "JsonRpc",
@@ -116,6 +119,7 @@ __all__ = [
     "is_otel_v2_enabled",
     "promoted_baggage",
     "resolve_operation",
+    "resolve_output_type",
     "resolve_provider",
     "span_role_for_service",
     "validate_registry",

litellm/integrations/otel/mappers/genai.py

@@ -42,6 +42,7 @@ class GenAIMapper:
     _LLM_CALL_ATTRS: dict[str, Callable[[LLMCallSpanData], AttrValue | None]] = {
         GenAI.OPERATION_NAME: lambda d: d.operation.value,
         GenAI.PROVIDER_NAME: lambda d: d.provider or None,
+        GenAI.OUTPUT_TYPE: lambda d: d.output_type.value if d.output_type else None,
         GenAI.REQUEST_MODEL: lambda d: d.request_model or None,
         GenAI.REQUEST_TEMPERATURE: lambda d: d.request_params.temperature,
         GenAI.REQUEST_TOP_P: lambda d: d.request_params.top_p,
@@ -65,6 +66,7 @@ class GenAIMapper:
         Server.ADDRESS: lambda d: d.server.address if d.server else None,
         Server.PORT: lambda d: d.server.port if d.server else None,
         LiteLLM.CALL_ID: lambda d: d.identity.call_id or None,
+        LiteLLM.CALL_TYPE: lambda d: d.call_type,
         # The provider/underlying model is only known once routing has picked a
         # deployment, so it can't ride identity Baggage (seeded at auth, before
         # routing) onto the boundary-born LLM span — stamp it directly here.

litellm/integrations/otel/model/payloads.py

@@ -15,8 +15,10 @@ from litellm.integrations.otel.model.metadata import (
 )
 from litellm.integrations.otel.model.semconv import (
     GenAIOperation,
+    GenAIOutputType,
     MCPMethod,
     resolve_operation,
+    resolve_output_type,
     resolve_provider,
 )
 from litellm.integrations.otel.model.utils import (
@@ -310,6 +312,11 @@ class LLMCallSpanData:
     choices_out: tuple[Mapping[str, object], ...] = ()
     system_fingerprint: str | None = None
     time_to_first_chunk_seconds: float | None = None
+    # The requested output modality, set only on the routes that pin one (image
+    # generation, speech, transcription, OCR), and the litellm route itself, which
+    # keeps routes the convention folds into one operation distinguishable.
+    output_type: GenAIOutputType | None = None
+    call_type: str | None = None
 
     @classmethod
     def from_standard_logging_payload(
@@ -334,8 +341,9 @@ class LLMCallSpanData:
         # otherwise the content-bearing mappers receive empty sequences and emit
         # no prompt/response text.
         finish_reasons: Final = _finish_reasons(choices_out)
+        call_type: Final = as_str(payload.get("call_type"))
         return cls(
-            operation=resolve_operation(as_str(payload.get("call_type"))),
+            operation=resolve_operation(call_type),
             provider=resolve_provider(as_str(payload.get("custom_llm_provider"))),
             request_model=context.request_model,
             response_model=context.response_model,
@@ -358,6 +366,8 @@ class LLMCallSpanData:
             choices_out=choices_out if capture_content else (),
             system_fingerprint=as_str(response.get("system_fingerprint")),
             time_to_first_chunk_seconds=time_to_first_chunk_seconds,
+            output_type=resolve_output_type(call_type),
+            call_type=call_type or None,
         )
 
 

litellm/integrations/otel/model/semconv.py

@@ -3,7 +3,9 @@ Keys follow the OpenTelemetry GenAI semantic conventions (experimental). Anythin
 without a semconv equivalent lives under the ``litellm.*`` vendor namespace.
 """
 
+from collections.abc import Mapping
 from enum import Enum
+from types import MappingProxyType
 from typing import Final
 
 from litellm._logging import verbose_logger
@@ -30,6 +32,21 @@ class GenAIOperation(str, Enum):
     EXECUTE_TOOL = "execute_tool"  # MCP tool-call spans
     LITELLM_VECTOR_STORE_MANAGEMENT = "litellm.vector_store_management"
     LITELLM_VECTOR_STORE_FILE_MANAGEMENT = "litellm.vector_store_file_management"
+    LITELLM_MODERATION = "litellm.moderation"
+
+
+class GenAIOutputType(str, Enum):
+    """Values for ``gen_ai.output.type``, the modality the client asked for.
+
+    It is what separates the inference routes that share ``generate_content``:
+    image generation requests ``image``, speech requests ``speech``, and
+    transcription and OCR both request ``text``.
+    """
+
+    TEXT = "text"
+    JSON = "json"
+    IMAGE = "image"
+    SPEECH = "speech"
 
 
 class GenAIProvider(str, Enum):
@@ -258,6 +275,11 @@ class LiteLLM:
     """Vendor-extension keys (no semconv equivalent). Always ``litellm.*``."""
 
     CALL_ID: Final = "litellm.call_id"
+    # The litellm route that produced the call. Needed because the convention maps
+    # several routes onto one operation: transcription and OCR are both
+    # ``generate_content`` with a ``text`` output type, so this is the only thing
+    # that tells them apart.
+    CALL_TYPE: Final = "litellm.call_type"
     COST_PREFIX: Final = "litellm.cost."
     METADATA_PREFIX: Final = "litellm.metadata."
     TEAM_ID: Final = "litellm.team.id"
@@ -352,6 +374,16 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
     "aembedding": GenAIOperation.EMBEDDINGS,
     "responses": GenAIOperation.CHAT,
     "aresponses": GenAIOperation.CHAT,
+    "image_generation": GenAIOperation.GENERATE_CONTENT,
+    "aimage_generation": GenAIOperation.GENERATE_CONTENT,
+    "moderation": GenAIOperation.LITELLM_MODERATION,
+    "amoderation": GenAIOperation.LITELLM_MODERATION,
+    "ocr": GenAIOperation.GENERATE_CONTENT,
+    "aocr": GenAIOperation.GENERATE_CONTENT,
+    "speech": GenAIOperation.GENERATE_CONTENT,
+    "aspeech": GenAIOperation.GENERATE_CONTENT,
+    "transcription": GenAIOperation.GENERATE_CONTENT,
+    "atranscription": GenAIOperation.GENERATE_CONTENT,
     "call_mcp_tool": GenAIOperation.EXECUTE_TOOL,
     "vector_store_search": GenAIOperation.RETRIEVAL,
     "avector_store_search": GenAIOperation.RETRIEVAL,
@@ -385,6 +417,23 @@ _OPERATION_BY_CALL_TYPE: Final[dict[str, GenAIOperation]] = {
 }
 
 
+# litellm ``call_type`` -> ``gen_ai.output.type``. Only the call types whose route
+# fixes the requested modality are listed; the attribute is conditionally required
+# on a request that asks for an output format, so anything else is left unstamped.
+_OUTPUT_TYPE_BY_CALL_TYPE: Final[Mapping[str, GenAIOutputType]] = MappingProxyType(
+    {
+        "image_generation": GenAIOutputType.IMAGE,
+        "aimage_generation": GenAIOutputType.IMAGE,
+        "speech": GenAIOutputType.SPEECH,
+        "aspeech": GenAIOutputType.SPEECH,
+        "transcription": GenAIOutputType.TEXT,
+        "atranscription": GenAIOutputType.TEXT,
+        "ocr": GenAIOutputType.TEXT,
+        "aocr": GenAIOutputType.TEXT,
+    }
+)
+
+
 def resolve_provider(custom_llm_provider: str | None) -> str:
     """Map a litellm provider string to a ``gen_ai.provider.name`` value.
 
@@ -416,3 +465,11 @@ def resolve_operation(call_type: str | None) -> GenAIOperation:
         GenAIOperation.CHAT.value,
     )
     return GenAIOperation.CHAT
+
+
+def resolve_output_type(call_type: str | None) -> GenAIOutputType | None:
+    """Map a litellm ``call_type`` to a ``gen_ai.output.type`` value, or ``None``
+    for a route that doesn't pin the output modality."""
+    if not call_type:
+        return None
+    return _OUTPUT_TYPE_BY_CALL_TYPE.get(call_type.lower())

litellm/integrations/prometheus.py

@@ -215,7 +215,9 @@ class PrometheusLogger(CustomLogger):
             # request latency metrics
             self.litellm_request_total_latency_metric = self._histogram_factory(
                 "litellm_request_total_latency_metric",
-                "Total latency (seconds) for a request to LiteLLM",
+                "End-to-end latency (seconds) for a request to LiteLLM Proxy Server, from the moment "
+                "the request reached the proxy through the end of processing -- includes "
+                "authentication, pre-call hooks, the LLM API call, and post-call processing",
                 labelnames=self.get_labels_for_metric("litellm_request_total_latency_metric"),
                 buckets=self.latency_buckets,
             )
@@ -458,7 +460,8 @@ class PrometheusLogger(CustomLogger):
             # Request queue time metric
             self.litellm_request_queue_time_metric = self._histogram_factory(
                 "litellm_request_queue_time_seconds",
-                "Time spent in request queue before processing starts (seconds)",
+                "Time (seconds) from request arrival at the proxy to the start of pre-call "
+                "processing -- includes authentication and any ASGI-level queueing",
                 labelnames=self.get_labels_for_metric("litellm_request_queue_time_seconds"),
                 buckets=self.latency_buckets,
             )
@@ -2078,27 +2081,37 @@ class PrometheusLogger(CustomLogger):
                 _labels,
             )
 
-        # total request latency
+        # request queue time (time from arrival to processing start) -- read first so
+        # it can be folded into the total-latency metric below. start_time/end_time
+        # only span from after auth completes, so without this the "total" latency
+        # metric silently excludes auth and pre-call hook time.
+        _litellm_params: Final = kwargs.get("litellm_params", {}) or {}
+        queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
+
+        # total request latency: true end-to-end, from request arrival (queue_time_seconds,
+        # when available) through the end of processing.
         total_time_seconds: Final = self._safe_duration_seconds(
             start_time=start_time,
             end_time=end_time,
         )
         if total_time_seconds is not None:
+            _observed_total_time_seconds: Final = (
+                total_time_seconds + queue_time_seconds
+                if queue_time_seconds is not None and queue_time_seconds >= 0
+                else total_time_seconds
+            )
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_total_latency_metric"),
                 enum_values=enum_values,
                 label_context=label_context,
             )
-            self.litellm_request_total_latency_metric.labels(**_labels).observe(total_time_seconds)
+            self.litellm_request_total_latency_metric.labels(**_labels).observe(_observed_total_time_seconds)
             self._track_end_user_metric_series(
                 self.litellm_request_total_latency_metric,
                 "litellm_request_total_latency_metric",
                 _labels,
             )
 
-        # request queue time (time from arrival to processing start)
-        _litellm_params: Final = kwargs.get("litellm_params", {}) or {}
-        queue_time_seconds: Final = (_litellm_params.get("metadata") or {}).get("queue_time_seconds")
         if queue_time_seconds is not None and queue_time_seconds >= 0:
             _labels = prometheus_label_factory(
                 supported_enum_labels=self.get_labels_for_metric(metric_name="litellm_request_queue_time_seconds"),

litellm/integrations/websearch_interception/ARCHITECTURE.md

@@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
 
 ---
 
+## Loop Ceiling
+
+One intercepted request can chain several follow-up model calls, since the model often searches again after
+reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
+to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
+
+Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
+
+```yaml
+litellm_settings:
+  websearch_interception_params:
+    enabled_providers: ["bedrock"]
+    max_agentic_loops: 5
+```
+
+Or per deployment, which wins over the feature-level setting:
+
+```yaml
+model_list:
+  - model_name: claude-sonnet-4-5
+    litellm_params:
+      model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
+      max_agentic_loops: 5
+```
+
+Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
+carries it is ignored and one request can never drive an unbounded number of upstream model calls.
+
+Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
+from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
+not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
+validator down there would drop the deployment silently instead of refusing to start.
+
+When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
+the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
+The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
+The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
+buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
+
+Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
+treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
+rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
+client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
+already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
+rather than as a replacement.
+
+Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
+not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
+rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
+Both are tracked separately
+
+---
+
 ## Streaming Support
 
 WebSearch interception works transparently with both streaming and non-streaming requests.

litellm/integrations/websearch_interception/handler.py

@@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
 from litellm.integrations.websearch_interception.transformation import (
     WebSearchTransformation,
 )
+from litellm.litellm_core_utils.agentic_loop_settings import (
+    validated_max_agentic_loops,
+)
 from litellm.llms.base_llm.search.transformation import SearchResponse
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
@@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         self,
         enabled_providers: list[LlmProviders | str] | None = None,
         search_tool_name: str | None = None,
+        max_agentic_loops: int | None = None,
     ):
         """
         Args:
@@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
                               Default: None (all providers enabled)
             search_tool_name: Name of search tool configured in router's search_tools.
                              If None, will attempt to use first available search tool.
+            max_agentic_loops: How many follow-up model calls one intercepted request
+                              may chain before the loop is refused and the turn ends.
+                              If None, LiteLLM's default of 3 applies.
         """
         super().__init__()
         # Convert enum values to strings for comparison
@@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
         else:
             self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
         self.search_tool_name = search_tool_name
+        self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
         self._request_has_websearch = False  # Track if current request has web search
 
+    @staticmethod
+    def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
+        """
+        Reject loop ceilings the agentic loop cannot honor, at config load time.
+        """
+        return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
+
     async def try_short_circuit_search(
         self,
         model: str,
@@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
                   websearch_interception_params:
                     enabled_providers: ["bedrock"]
                     search_tool_name: "my-perplexity-search"
+                    max_agentic_loops: 5
 
             Usage:
                 config = litellm_settings.get("websearch_interception_params", {})
@@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         # Extract parameters from config
         enabled_providers_str: Final = config.get("enabled_providers", None)
         search_tool_name: Final = config.get("search_tool_name", None)
+        max_agentic_loops: Final = config.get("max_agentic_loops", None)
 
         # Convert string provider names to LlmProviders enum values
         enabled_providers: list[LlmProviders | str] | None = None
@@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         return cls(
             enabled_providers=enabled_providers,
             search_tool_name=search_tool_name,
+            max_agentic_loops=max_agentic_loops,
         )
 
     @staticmethod
@@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
 
         verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
 
+        deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
+        if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
+            kwargs["max_agentic_loops"] = self.max_agentic_loops  # rebind-ok: this hook returns the kwargs it edits
+
         # If the client sent an Anthropic-native web_search_* tool, mark the
         # request so the agentic loop emits native web_search_tool_result
         # blocks in the final response (for citations panels, etc.). The flag

litellm/litellm_core_utils/agentic_loop_settings.py

@@ -0,0 +1,59 @@
+"""
+Shared validation for the agentic loop ceiling.
+
+``max_agentic_loops`` can be set in two places, and the two disagreed about
+what a bad value means. The feature-level
+``litellm_settings.websearch_interception_params.max_agentic_loops`` was
+checked at config load, while a per-deployment
+``model_list[].litellm_params.max_agentic_loops`` was passed straight through
+to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
+turning the tightest ceiling into the loosest one, and let a per-deployment
+``"three"`` boot the proxy and then fail every request to that model.
+
+Both settings now go through :func:`validated_max_agentic_loops`, which names
+the field it rejected so the error says which line of the config to fix.
+
+Anything that spells a whole number is still accepted, because the old
+``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
+``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
+string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
+"""
+
+from typing import Final
+
+DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
+
+
+def _as_whole_number(value: object) -> int | None:
+    """
+    Return ``value`` as an int when it spells a whole number, else ``None``.
+
+    ``bool`` is excluded explicitly because it is an ``int`` subclass, so
+    ``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
+    """
+    if isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value) if value.is_integer() else None
+    if isinstance(value, str):
+        try:
+            return int(value.strip())
+        except ValueError:
+            return None
+    return None
+
+
+def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
+    """
+    Return ``max_agentic_loops`` as an int, or raise naming ``field``.
+    """
+    if max_agentic_loops is None:
+        return None
+    ceiling: Final = _as_whole_number(max_agentic_loops)
+    if ceiling is None:
+        raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
+    if ceiling < 1:
+        raise ValueError(f"{field} must be at least 1, got {ceiling}")
+    return ceiling

litellm/litellm_core_utils/chat_completion_agentic_loop.py

@@ -5,6 +5,10 @@ from typing import Final, cast
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.agentic_loop_settings import (
+    DEFAULT_MAX_AGENTIC_LOOPS,
+    validated_max_agentic_loops,
+)
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
     NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
 
 def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
     depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
-    max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
+    configured: Final = validated_max_agentic_loops(
+        kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
+    )
+    max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
     raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
     fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
     return depth, max_loops, fingerprints

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -811,6 +811,24 @@ def _map_openai_like_exception(
             )
 
 
+_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
+
+
+def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
+    """
+    Mantle reports context overflow as a structured validation error rather than
+    the plain-text patterns Bedrock itself uses, so it needs its own detection and a
+    message clients recognize as context overflow (litellm/litellm#36546).
+    """
+    if "invalid_request_error" not in error_str and "validation_error" not in error_str:
+        return None
+    match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
+    if match is None:
+        return None
+    prompt_tokens, max_tokens = match.groups()
+    return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
+
+
 def _map_bedrock_exception(
     *,
     model: str,
@@ -821,6 +839,14 @@ def _map_bedrock_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    if custom_llm_provider == "bedrock_mantle":
+        mantle_context_window_message = _get_bedrock_mantle_context_window_message(error_str)
+        if mantle_context_window_message is not None:
+            raise ContextWindowExceededError(
+                message=mantle_context_window_message,
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
     if (
         "too many tokens" in error_str
         or "expected maxLength:" in error_str
@@ -2315,7 +2341,7 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
-            elif custom_llm_provider == "bedrock":
+            elif custom_llm_provider in ("bedrock", "bedrock_mantle"):
                 _map_bedrock_exception(
                     model=model,
                     original_exception=mappable_exception,

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -1,3 +1,4 @@
+from collections.abc import Sequence
 from typing import Final
 
 from litellm.types.utils import ProviderSpecificHeader
@@ -6,13 +7,17 @@ from litellm.types.utils import ProviderSpecificHeader
 class ProviderSpecificHeaderUtils:
     @staticmethod
     def get_provider_specific_headers(
-        provider_specific_header: ProviderSpecificHeader | None,
+        provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
         custom_llm_provider: str | None,
     ) -> dict:
         """
         Get the provider specific headers for the given custom llm provider.
 
-        Supports comma-separated provider lists for headers that work across multiple providers.
+        Accepts either a single ProviderSpecificHeader or a sequence of them. Each entry
+        carries its own comma-separated provider list, so headers that are safe for several
+        providers and headers that are safe for exactly one can travel on the same request
+        without sharing a scope. Entries whose provider list does not contain
+        `custom_llm_provider` contribute nothing.
 
         Returns:
             Dict: The provider specific headers for the given custom llm provider
@@ -20,10 +25,15 @@ class ProviderSpecificHeaderUtils:
         if provider_specific_header is None or custom_llm_provider is None:
             return {}
 
-        stored_providers: Final = provider_specific_header.get("custom_llm_provider", "")
-        provider_list: Final = [p.strip() for p in stored_providers.split(",")]
+        scoped_headers: Final = (
+            (provider_specific_header,) if isinstance(provider_specific_header, dict) else provider_specific_header
+        )
 
-        if custom_llm_provider in provider_list:
-            return provider_specific_header.get("extra_headers", {})
+        matched_headers: Final = {}
+        for scoped_header in scoped_headers:
+            stored_providers = scoped_header.get("custom_llm_provider", "")
+            provider_list = [p.strip() for p in stored_providers.split(",")]
+            if custom_llm_provider in provider_list:
+                matched_headers.update(scoped_header.get("extra_headers", {}))
 
-        return {}
+        return matched_headers

litellm/litellm_core_utils/litellm_logging.py

@@ -5615,6 +5615,37 @@ def _extract_response_obj_and_hidden_params(
     return response_obj, hidden_params
 
 
+def _autorouter_savings_for_payload(
+    request_metadata: Mapping[str, object],
+    model: str | None,
+    custom_llm_provider: str | None,
+    model_id: str | None,
+    usage_object: Mapping[str, object] | None,
+    cost_breakdown: Mapping[str, object] | None,
+) -> float | None:
+    """The auto-router savings figure for the payload, or ``None`` when there is none.
+
+    Lazy proxy import: the savings module lives with the spend trackers that own the
+    math, and SDK-only installs have no proxy package to import.
+    """
+    try:
+        from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload
+    except Exception:  # noqa: BLE001  # SDK-only install: no savings driver to run
+        return None
+    try:
+        return autorouter_savings_for_logging_payload(
+            request_metadata=request_metadata,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            model_id=model_id,
+            usage_object=usage_object,
+            cost_breakdown=cost_breakdown,
+        )
+    except Exception as e:  # noqa: BLE001  # a savings figure must never fail request logging
+        verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
+        return None
+
+
 def get_standard_logging_object_payload(
     kwargs: dict | None,
     init_response_obj: Any | BaseModel | dict,
@@ -5772,6 +5803,16 @@ def get_standard_logging_object_payload(
         ):
             model_name = response_model_name
 
+        request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
+        autorouter_savings: Final = _autorouter_savings_for_payload(
+            request_metadata=metadata,
+            model=model_name,
+            custom_llm_provider=custom_llm_provider,
+            model_id=_model_id,
+            usage_object=usage_dict,
+            cost_breakdown=request_cost_breakdown,
+        )
+
         payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
             id=str(id),
             litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
@@ -5802,7 +5843,8 @@ def get_standard_logging_object_payload(
             metadata=clean_metadata,
             cache_key=clean_hidden_params["cache_key"],
             response_cost=response_cost,
-            cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
+            cost_breakdown=request_cost_breakdown,
+            autorouter_savings=autorouter_savings,
             total_tokens=usage_dict.get("total_tokens", 0),
             prompt_tokens=usage_dict.get("prompt_tokens", 0),
             completion_tokens=usage_dict.get("completion_tokens", 0),
@@ -5998,6 +6040,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         call_type="completion",
         stream=False,
         response_cost=response_cost,
+        autorouter_savings=None,
         response_cost_failure_debug_info=None,
         status="success",
         total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT),

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1371,6 +1371,7 @@ class CostCalculatorUtils:
             return fal_ai_image_cost_calculator(
                 model=model,
                 image_response=completion_response,
+                optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.RUNWAYML.value:
             from litellm.llms.runwayml.cost_calculator import (

litellm/litellm_core_utils/logging_utils.py

@@ -3,6 +3,7 @@ import functools
 import inspect
 import re
 import time
+from collections.abc import Mapping
 from datetime import datetime
 from typing import TYPE_CHECKING, Any, Final
 
@@ -268,6 +269,16 @@ def _set_duration_in_model_call_details(
         verbose_logger.warning("Error setting `llm_api_duration_ms`: %s", e)
 
 
+def speech_request_body(model: str, voice: str, optional_params: Mapping[str, object]) -> Mapping[str, object]:
+    """Speech request body for telemetry, without the caller headers the provider SDKs
+    take as request kwargs rather than body fields."""
+    return {  # mutable-ok: loggers isinstance-check the request body as a dict
+        "model": model,
+        "voice": voice,
+        **{key: value for key, value in optional_params.items() if key != "extra_headers"},
+    }
+
+
 def track_llm_api_timing():
     """
     Decorator to track LLM API call timing for both sync and async functions.

litellm/litellm_core_utils/ptu_pricing.py

@@ -8,7 +8,7 @@ router prices at zero serves its traffic for free.
 
 from collections.abc import Mapping
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import date, datetime, time, timezone
 from types import MappingProxyType
 from typing import Final
 
@@ -68,9 +68,17 @@ def _to_utc(parsed: datetime) -> datetime:
 
 
 def _as_utc(value: object) -> datetime | None:
-    """A model_info datetime as UTC, parsing an ISO string, else None."""
+    """A model_info datetime as UTC, parsing an ISO string, else None.
+
+    An unquoted ``2027-01-01`` in config.yaml is loaded as a ``date``, not a string, and a
+    reservation bound that fails to parse takes the whole deployment out of PTU handling,
+    so the day is read as its opening midnight rather than discarded. ``datetime`` derives
+    from ``date``, so it has to be matched first.
+    """
     if isinstance(value, datetime):
         return _to_utc(value)
+    if isinstance(value, date):
+        return datetime.combine(value, time.min, tzinfo=timezone.utc)
     if not isinstance(value, str):
         return None
     try:
@@ -84,6 +92,46 @@ def _named(reason: str, model_name: str | None) -> str:
     return reason if model_name is None else f"PTU configuration on model '{model_name}' is invalid: {reason}"
 
 
+def ptu_identity_error(
+    *, declared_id: str | None, taken: bool, current_id: str | None = None, model_name: str | None = None
+) -> str | None:
+    """Why this config-declared reservation cannot be identified, else None.
+
+    A deployment declared in config.yaml is otherwise keyed by a hash of its resolved
+    ``litellm_params``, so rotating a credential or editing an endpoint mints a second
+    identity and the reservation is charged again under it. The flat cost is keyed by that
+    id, and a charge already written is never retracted, so the duplicate is permanent.
+
+    ``current_id`` is what the deployment is keyed by today. Naming it is the difference
+    between an operator carrying their history forward and an operator inventing a fresh
+    id, which starts a second identity beside the charges already written.
+    """
+    if not declared_id:
+        return _named(
+            "model_info.id is required when PTU fields are set. Without one the deployment is "
+            "identified by a hash of its litellm_params, so rotating a credential bills the "
+            "reservation a second time under the new identity. Set it to the id this deployment "
+            f"already uses, {current_id or 'shown by GET /model/info'}, so the flat cost already "
+            "written stays under one identity; any other value starts a second one",
+            model_name,
+        )
+    if taken:
+        return _named(
+            f"model_info.id '{declared_id}' is declared on more than one deployment. Each would key "
+            "the same flat-cost row, so one reservation would go unbilled",
+            model_name,
+        )
+    return None
+
+
+PTU_MODEL_INFO_FIELDS: Final = ("ptu_count", "cost_per_ptu_per_hour", "ptu_effective_from", "ptu_effective_to")
+
+
+def declares_ptu(model_info: Mapping[str, object]) -> bool:
+    """Whether any PTU field is set here, including one too malformed to charge."""
+    return any(model_info.get(field) is not None for field in PTU_MODEL_INFO_FIELDS)
+
+
 def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
     """Why this PTU configuration cannot be honoured, else None.
 

litellm/litellm_core_utils/streaming_handler.py

@@ -191,7 +191,7 @@ class CustomStreamWrapper:
         custom_llm_provider: str | None = None,
         stream_options=None,
         make_call: Callable | None = None,
-        _response_headers: dict | None = None,
+        _response_headers: dict | httpx.Headers | None = None,
     ):
         self.model = model
         self.make_call = make_call
@@ -2315,10 +2315,18 @@ class CustomStreamWrapper:
         if self.logging_obj is None or not self.chunks:
             return
         try:
-            partial_response: Final = litellm.stream_chunk_builder(chunks=self.chunks)
+            partial_response: Final = litellm.stream_chunk_builder(
+                chunks=self.chunks,
+                messages=self.messages if isinstance(self.messages, list) else None,
+            )
+            if partial_response is None:
+                return
             usage: Final = cast(Usage | None, getattr(partial_response, "usage", None))
             if usage is None:
                 return
+            if self.model:
+                partial_response.model = self.model
+            backfill_missing_cache_usage_fields(usage)
             self.logging_obj.model_call_details["combined_usage_object"] = usage
             self.logging_obj.model_call_details["response_cost"] = (
                 self.logging_obj._response_cost_calculator(result=partial_response) or 0.0
@@ -2439,6 +2447,35 @@ class CustomStreamWrapper:
         return chunk
 
 
+def _cache_token_count(details: PromptTokensDetailsWrapper | None, keys: tuple[str, ...]) -> int:
+    for key in keys:
+        value = getattr(details, key, None)
+        if isinstance(value, int) and not isinstance(value, bool) and value:
+            return value
+    return 0
+
+
+def backfill_missing_cache_usage_fields(usage: Usage) -> None:
+    """Give partial-stream usage the same cache fields a complete stream reports.
+
+    Carries OpenAI-style ``prompt_tokens_details`` counts up to the Anthropic-style
+    top-level keys, defaulting to zero. It must carry the real count rather than a
+    flat zero: downstream readers treat these keys as authoritative once present and
+    skip their own normalization, so a zero here would overwrite a real cache read.
+    """
+    details: Final = usage.prompt_tokens_details
+    if getattr(usage, "cache_read_input_tokens", None) is None:
+        usage.cache_read_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cached_tokens",)
+        )
+    if getattr(usage, "cache_creation_input_tokens", None) is None:
+        usage.cache_creation_input_tokens = _cache_token_count(  # rebind-ok: in-place backfill is the contract
+            details, ("cache_write_tokens", "cache_creation_tokens")
+        )
+    if usage.prompt_tokens_details is None:
+        usage.prompt_tokens_details = PromptTokensDetailsWrapper(cached_tokens=0)  # rebind-ok: backfill in place
+
+
 _TokenDetails = TypeVar("_TokenDetails", PromptTokensDetailsWrapper, CompletionTokensDetailsWrapper)
 
 

litellm/llms/anthropic/chat/transformation.py

@@ -1827,6 +1827,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             custom_llm_provider=self.custom_llm_provider,
         )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=optional_params,
+            custom_llm_provider=self._resolved_provider,
+        )
+
         headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
 
         # === Tool-name sanitization (single chokepoint) ===
@@ -2216,7 +2222,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
     def calculate_usage(
         self,
-        usage_object: dict,
+        usage_object: Mapping[str, Any],
         reasoning_content: str | None,
         completion_response: dict | None = None,
         speed: str | None = None,

litellm/llms/anthropic/common_utils.py

@@ -32,6 +32,12 @@ from litellm.types.llms.anthropic import (
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.proxy.model_listing import ModelInfoResponse
 
+DROP_DISABLED_THINKING_WARNING: Final = (
+    "Dropping `thinking={'type': 'disabled'}` for model=%s: thinking is always on for this model and cannot be "
+    "disabled (the alternative is a provider 400). The model will still think adaptively, its response can contain "
+    "thinking blocks, and those thinking tokens are billed as output tokens."
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
@@ -425,6 +431,35 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         """
         return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider)
 
+    @staticmethod
+    def _is_always_on_thinking_model(model: str, custom_llm_provider: str) -> bool:
+        """Whether ``model`` always thinks and rejects ``thinking.type=disabled``
+        (Fable 5 / Mythos 5 generation). The model cost map is authoritative: an
+        explicit ``thinking_always_on`` entry resolved under ``custom_llm_provider``,
+        or a ``fallback_generalizations`` rule for unmapped ids of those families.
+        """
+        return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
+
+    @staticmethod
+    def maybe_drop_disabled_thinking(
+        model: str,
+        optional_params: dict,  # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
+        custom_llm_provider: str,
+    ) -> None:
+        """Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models
+        (Fable 5 / Mythos 5), which 400 on it; omission is the API-documented
+        remedy and yields the model's default adaptive thinking."""
+        thinking: Final = optional_params.get("thinking")
+        if not isinstance(thinking, dict) or thinking.get("type") != "disabled":
+            return
+        if not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider):
+            return
+        litellm.verbose_logger.warning(
+            DROP_DISABLED_THINKING_WARNING,
+            model,
+        )
+        optional_params.pop("thinking", None)
+
     def is_effort_used(
         self,
         optional_params: dict | None,

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
 )
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
+    local_model_name,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         except Exception:
             pass
 
-        if isinstance(model, str) and model and not model.startswith("responses/"):
-            # Prefix model with "responses/" to route to OpenAI Responses API
-            completion_kwargs["model"] = f"responses/{model}"
+        if isinstance(model, str) and model and "responses/" not in model:
+            local_model: Final = model.removeprefix(f"{custom_llm_provider}/")
+            completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}"
 
         auto_summary: Final = is_reasoning_auto_summary_enabled()
 
@@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         if stream:
             transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
                 completion_response,
-                model=model,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=True,
@@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         if stream:
             transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
                 completion_response,
-                model=model,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=False,

litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py

@@ -56,7 +56,7 @@ from ..result import PolyfillResult
 # so the summary's spend is attributed to the same scopes. The list mirrors the
 # fields populated by
 # ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata``.
-# ``user_api_key_model_max_budget`` / ``user_api_key_end_user_model_max_budget``
+# The three ``*_model_max_budget`` fields
 # are what ``_PROXY_VirtualKeyModelMaxBudgetLimiter`` reads post-call to update
 # the per-model spend caches, so without them the summary spend would never
 # count against the caller's model budget. ``user_api_key_end_user_id`` /
@@ -76,6 +76,7 @@ _PROPAGATED_METADATA_KEYS: Final = (
     "user_api_key_end_user_id",
     "user_api_end_user_max_budget",
     "user_api_key_model_max_budget",
+    "user_api_key_user_model_max_budget",
     "user_api_key_end_user_model_max_budget",
     "litellm_call_id",
     "litellm_parent_otel_span",
@@ -317,10 +318,14 @@ async def _check_summary_model_budget(
     The summary subrequest never passes back through ``user_api_key_auth``, so
     without this gate a caller whose ``model_max_budget`` for
     ``context_management_summary_model`` is exhausted could keep consuming that
-    model via compaction. Mirrors the ``model_max_budget`` /
-    ``end_user_model_max_budget`` enforcement that ``user_api_key_auth`` runs for
-    the client-requested model. Returns True outside the proxy or when no
+    model via compaction. Mirrors the per-model budget enforcement that
+    ``user_api_key_auth`` runs for the client-requested model. Returns True outside the proxy or when no
     per-model budget is configured.
+
+    All three scopes are checked because the summary's spend is charged to all
+    three: this file propagates the key, user and end-user budgets into the
+    subrequest's metadata, so enforcing only two of them would let compaction
+    increment a counter it can never be refused by.
     """
     if user_api_key_auth is None:
         return True
@@ -347,6 +352,25 @@ async def _check_summary_model_budget(
             )
             return False
 
+    user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
+    user_id: Final = getattr(user_api_key_auth, "user_id", None)
+    if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
+        try:
+            await model_max_budget_limiter.is_user_within_model_budget(
+                user_id=user_id,
+                user_model_max_budget=user_model_max_budget,
+                model=summary_model,
+            )
+        except litellm.BudgetExceededError:
+            return False
+        except Exception as e:  # noqa: BLE001  # a budget gate denies on any failure, as the key and end-user scopes do
+            verbose_logger.warning(
+                "compact_20260112: unexpected error during user model-budget check for summary_model=%s; denying: %s",
+                summary_model,
+                e,
+            )
+            return False
+
     end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None)
     end_user_id: Final = getattr(user_api_key_auth, "end_user_id", None)
     if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None:

litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py

@@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator:
             }
             chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
 
+        else:
+            passthrough_start: Final = {
+                "type": "content_block_start",
+                "index": index,
+                "content_block": block_dict,
+            }
+            chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode())
+
         content_block_stop: Final = {"type": "content_block_stop", "index": index}
         chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode())
         return chunks

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response
 _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
 
 
-def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool:
-    """Return True when the provider should use the Responses API path.
+def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
+    from litellm.main import responses_api_bridge_check
+
+    model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
+    return model_info.get("mode") == "responses"
+
+
+def _responses_mode_is_lost_by_prefix_strip(
+    requested_model: str, resolved_model: str, custom_llm_provider: str
+) -> bool:
+    """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
+
+    ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
+    deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
+    chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
+    """
+    if requested_model == resolved_model:
+        return False
+    return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
+        resolved_model, custom_llm_provider
+    )
+
+
+def _should_route_to_responses_api(
+    custom_llm_provider: str | None,
+    requested_model: str | None = None,
+    resolved_model: str | None = None,
+) -> bool:
+    """Return True when the request should use the Responses API path.
 
     Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
     opt out and route OpenAI/Azure requests through chat/completions instead.
     """
     if litellm.use_chat_completions_url_for_anthropic_messages:
         return False
-    return custom_llm_provider in _RESPONSES_API_PROVIDERS
+    if custom_llm_provider in _RESPONSES_API_PROVIDERS:
+        return True
+    if custom_llm_provider is None or requested_model is None or resolved_model is None:
+        return False
+    return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
 
 
 def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
@@ -533,7 +564,7 @@ def anthropic_messages_handler(
         _shared_kwargs: Final = dict(
             max_tokens=max_tokens,
             messages=messages,
-            model=model,
+            model=original_model,
             metadata=metadata,
             stop_sequences=stop_sequences,
             stream=stream,
@@ -551,7 +582,7 @@ def anthropic_messages_handler(
             custom_llm_provider=custom_llm_provider,
             **kwargs,
         )
-        if _should_route_to_responses_api(custom_llm_provider):
+        if _should_route_to_responses_api(custom_llm_provider, original_model, model):
             return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs)
 
         # The in-gateway context_management polyfill runs inside

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -568,6 +568,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             custom_llm_provider=self._resolved_provider,
         )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=anthropic_messages_optional_request_params,
+            custom_llm_provider=self._resolved_provider,
+        )
+
         self._translate_legacy_thinking_for_adaptive_model(
             model=model,
             optional_params=anthropic_messages_optional_request_params,

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -44,6 +44,7 @@ class AnthropicMessagesRequestUtils:
         filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None}
         if model is not None:
             from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+            from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 
             AnthropicConfig._maybe_drop_speed_param(
                 model=model,
@@ -51,6 +52,16 @@ class AnthropicMessagesRequestUtils:
                 drop_params=drop_params,
                 custom_llm_provider=custom_llm_provider,
             )
+            for param in ("temperature", "top_p", "top_k"):
+                if param in filtered_params:
+                    AnthropicModelInfo._apply_sampling_param(  # pyright: ignore[reportPrivateUsage]  # same gating the /chat/completions path applies; forking it would drift
+                        optional_params=filtered_params,
+                        model=model,
+                        param=param,
+                        value=filtered_params.pop(param),
+                        drop_params=drop_params,
+                        output_key=param,
+                    )
         return cast(AnthropicMessagesRequestOptionalParams, filtered_params)
 
 

litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py

@@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
 
+from ..utils import local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
@@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         result: Final = await litellm.aresponses(**responses_kwargs)
 
         if stream:
-            wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
+            wrapper: Final = AnthropicResponsesStreamWrapper(
+                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+            )
             return wrapper.async_anthropic_sse_wrapper()
 
         if not isinstance(result, ResponsesAPIResponse):
@@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         result: Final = litellm.responses(**responses_kwargs)
 
         if stream:
-            wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
+            wrapper: Final = AnthropicResponsesStreamWrapper(
+                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+            )
             return wrapper.async_anthropic_sse_wrapper()
 
         if not isinstance(result, ResponsesAPIResponse):

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
     return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
 
 
+def local_model_name(model: str, custom_llm_provider: object) -> str:
+    """The id the provider itself knows, for reporting back to the caller in ``message_start``."""
+    return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model
+
+
 def is_reasoning_auto_summary_enabled() -> bool:
     """Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
     return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"

litellm/llms/azure/azure.py

@@ -17,7 +17,7 @@ from openai import (
 import litellm
 from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
 from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -1352,6 +1352,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         organization: str | None,
         max_retries: int,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
         azure_ad_token: str | None = None,
         azure_ad_token_provider: Callable | None = None,
         aspeech: bool | None = None,
@@ -1373,6 +1374,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 azure_ad_token_provider=azure_ad_token_provider,
                 max_retries=max_retries,
                 timeout=timeout,
+                logging_obj=logging_obj,
                 client=client,
                 litellm_params=litellm_params,
             )
@@ -1387,6 +1389,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             litellm_params=litellm_params,
         )
 
+        logging_obj.pre_call(
+            input=input,
+            api_key=api_key,
+            additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                "complete_input_dict": speech_request_body(model, voice, optional_params),
+                "api_base": str(azure_client.base_url),
+            },
+        )
+
         response: Final = azure_client.audio.speech.create(
             model=model,
             voice=voice,
@@ -1408,6 +1419,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         azure_ad_token_provider: Callable | None,
         max_retries: int,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
         client=None,
         litellm_params: dict | None = None,
     ) -> HttpxBinaryResponseContent:
@@ -1421,6 +1433,15 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             litellm_params=litellm_params,
         )
 
+        logging_obj.pre_call(
+            input=input,
+            api_key=api_key,
+            additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                "complete_input_dict": speech_request_body(model, voice, optional_params),
+                "api_base": str(azure_client.base_url),
+            },
+        )
+
         azure_response: Final = await azure_client.audio.speech.create(
             model=model,
             voice=voice,

litellm/llms/base_llm/files/transformation.py

@@ -11,9 +11,9 @@ from litellm.types.llms.openai import (
     AllMessageValues,
     CreateFileRequest,
     FileContentRequest,
+    FileListPage,
     OpenAICreateFileRequestOptionalParams,
     OpenAIFileObject,
-    OpenAIFilesPurpose,
 )
 from litellm.types.utils import LlmProviders, ModelResponse
 
@@ -240,10 +240,13 @@ class BaseFileEndpoints(ABC):
     @abstractmethod
     async def afile_list(
         self,
-        purpose: OpenAIFilesPurpose | None,
+        purpose: str | None,
         litellm_parent_otel_span: Span | None,
+        user_api_key_dict: UserAPIKeyAuth,
+        limit: int | None = None,
+        after: str | None = None,
         **data: dict,
-    ) -> list[OpenAIFileObject]:
+    ) -> FileListPage:
         pass
 
     @abstractmethod

litellm/llms/bedrock/chat/converse_handler.py

@@ -35,7 +35,7 @@ def make_sync_call(
     json_mode: bool | None = False,
     fake_stream: bool = False,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     if client is None:
         client = _get_httpx_client()  # Create a new client if none provided
 
@@ -76,7 +76,7 @@ def make_sync_call(
         additional_args={"complete_input_dict": data},
     )
 
-    return completion_stream
+    return completion_stream, response.headers
 
 
 class BedrockConverseLLM(BaseAWSLLM):
@@ -134,7 +134,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             },
         )
 
-        completion_stream: Final = await make_call(
+        completion_stream, response_headers = await make_call(
             client=client,
             api_base=api_base,
             headers=dict(prepped.headers),
@@ -151,6 +151,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 
@@ -232,7 +233,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return litellm.AmazonConverseConfig()._transform_response(
+        transformed_response: Final = litellm.AmazonConverseConfig()._transform_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -244,6 +245,8 @@ class BedrockConverseLLM(BaseAWSLLM):
             optional_params=optional_params,
             encoding=encoding,
         )
+        transformed_response.set_provider_response_headers(response.headers)
+        return transformed_response
 
     def completion(
         self,
@@ -541,7 +544,7 @@ class BedrockConverseLLM(BaseAWSLLM):
             client = client
 
         if stream is not None and stream is True:
-            completion_stream: Final = make_sync_call(
+            completion_stream, response_headers = make_sync_call(
                 client=(client if client is not None and isinstance(client, HTTPHandler) else None),
                 api_base=proxy_endpoint_url,
                 headers=prepped.headers,
@@ -558,6 +561,7 @@ class BedrockConverseLLM(BaseAWSLLM):
                 model=model,
                 custom_llm_provider="bedrock",
                 logging_obj=logging_obj,
+                _response_headers=response_headers,
             )
 
             return streaming_response
@@ -578,7 +582,7 @@ class BedrockConverseLLM(BaseAWSLLM):
         except httpx.TimeoutException:
             raise BedrockError(status_code=408, message="Timeout error occurred.")
 
-        return litellm.AmazonConverseConfig()._transform_response(
+        sync_transformed_response: Final = litellm.AmazonConverseConfig()._transform_response(
             model=model,
             response=response,
             model_response=model_response,
@@ -590,3 +594,5 @@ class BedrockConverseLLM(BaseAWSLLM):
             optional_params=optional_params,
             encoding=encoding,
         )
+        sync_transformed_response.set_provider_response_headers(response.headers)
+        return sync_transformed_response

litellm/llms/bedrock/chat/converse_transformation.py

@@ -39,6 +39,7 @@ from litellm.llms.anthropic.chat.transformation import (
     REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT,
     AnthropicConfig,
 )
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.request_metadata import (
     bedrock_request_metadata_headers,
@@ -1571,6 +1572,12 @@ class AmazonConverseConfig(BaseConfig):
                     "has no thinking_blocks. The model won't use extended thinking for this turn."
                 )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=optional_params,
+            custom_llm_provider="bedrock",
+        )
+
         # Prepare and separate parameters
         (
             inference_params,

litellm/llms/bedrock/chat/invoke_handler.py

@@ -163,7 +163,7 @@ async def make_call(
     json_mode: bool | None = False,
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     try:
         if client is None:
             client = get_async_httpx_client(
@@ -225,7 +225,7 @@ async def make_call(
             additional_args={"complete_input_dict": data},
         )
 
-        return completion_stream
+        return completion_stream, response.headers
     except httpx.HTTPStatusError as err:
         error_code: Final = err.response.status_code
         raise BedrockError(status_code=error_code, message=err.response.text)
@@ -248,7 +248,7 @@ def make_sync_call(
     json_mode: bool | None = False,
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None = None,
     stream_chunk_size: int | None = None,
-):
+) -> tuple[Any, httpx.Headers]:
     try:
         if client is None:
             client = _get_httpx_client(
@@ -309,7 +309,7 @@ def make_sync_call(
             additional_args={"complete_input_dict": data},
         )
 
-        return completion_stream
+        return completion_stream, response.headers
     except httpx.HTTPStatusError as err:
         error_code: Final = err.response.status_code
         raise BedrockError(status_code=error_code, message=err.response.text)

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -1,7 +1,6 @@
 import copy
 import json
 import time
-from functools import partial
 from typing import TYPE_CHECKING, Any, Final, cast, get_args
 
 import httpx
@@ -446,24 +445,24 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
     ) -> CustomStreamWrapper:
+        completion_stream, response_headers = await make_call(
+            client=client,
+            api_base=api_base,
+            headers=headers,
+            data=json.dumps(data),
+            model=model,
+            messages=messages,
+            logging_obj=logging_obj,
+            fake_stream=True if "ai21" in api_base else False,
+            bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
+            json_mode=json_mode,
+        )
         streaming_response: Final = CustomStreamWrapper(
-            completion_stream=None,
-            make_call=partial(
-                make_call,
-                client=client,
-                api_base=api_base,
-                headers=headers,
-                data=json.dumps(data),
-                model=model,
-                messages=messages,
-                logging_obj=logging_obj,
-                fake_stream=True if "ai21" in api_base else False,
-                bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
-                json_mode=json_mode,
-            ),
+            completion_stream=completion_stream,
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 
@@ -481,27 +480,28 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         json_mode: bool | None = None,
         signed_json_body: bytes | None = None,
     ) -> CustomStreamWrapper:
-        if client is None or isinstance(client, AsyncHTTPHandler):
-            client = _get_httpx_client(params={})
+        sync_client: Final = (
+            _get_httpx_client(params={}) if client is None or isinstance(client, AsyncHTTPHandler) else client
+        )
+        completion_stream, response_headers = make_sync_call(
+            client=sync_client,
+            api_base=api_base,
+            headers=headers,
+            data=json.dumps(data),
+            signed_json_body=signed_json_body,
+            model=model,
+            messages=messages,
+            logging_obj=logging_obj,
+            fake_stream=True if "ai21" in api_base else False,
+            bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
+            json_mode=json_mode,
+        )
         streaming_response: Final = CustomStreamWrapper(
-            completion_stream=None,
-            make_call=partial(
-                make_sync_call,
-                client=client,
-                api_base=api_base,
-                headers=headers,
-                data=json.dumps(data),
-                signed_json_body=signed_json_body,
-                model=model,
-                messages=messages,
-                logging_obj=logging_obj,
-                fake_stream=True if "ai21" in api_base else False,
-                bedrock_invoke_provider=self.get_bedrock_invoke_provider(model),
-                json_mode=json_mode,
-            ),
+            completion_stream=completion_stream,
             model=model,
             custom_llm_provider="bedrock",
             logging_obj=logging_obj,
+            _response_headers=response_headers,
         )
         return streaming_response
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -19,6 +19,10 @@ import litellm.types.utils
 from litellm._logging import _redact_string, verbose_logger
 from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
+from litellm.litellm_core_utils.agentic_loop_settings import (
+    DEFAULT_MAX_AGENTIC_LOOPS,
+    validated_max_agentic_loops,
+)
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
@@ -89,6 +93,7 @@ from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadCon
 from litellm.types.integrations.custom_logger import (
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
+    AgenticLoopSafetyError,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -635,6 +640,7 @@ class BaseLLMHTTPHandler:
                 model=model,
                 custom_llm_provider=custom_llm_provider,
                 logging_obj=logging_obj,
+                _response_headers=headers,
             )
 
         if client is None or not isinstance(client, HTTPHandler):
@@ -798,6 +804,7 @@ class BaseLLMHTTPHandler:
             model=model,
             custom_llm_provider=custom_llm_provider,
             logging_obj=logging_obj,
+            _response_headers=_response_headers,
         )
         return streamwrapper
 
@@ -2054,7 +2061,7 @@ class BaseLLMHTTPHandler:
         # Prepare headers
         kwargs = kwargs or {}
         provider_specific_header: Final = cast(
-            litellm.types.utils.ProviderSpecificHeader | None,
+            litellm.types.utils.ProviderSpecificHeader | Sequence[litellm.types.utils.ProviderSpecificHeader] | None,
             kwargs.get("provider_specific_header", None),
         )
         provider_specific_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
@@ -5075,9 +5082,12 @@ class BaseLLMHTTPHandler:
     @staticmethod
     def _get_agentic_loop_settings(kwargs: dict) -> tuple[int, int, list[str]]:
         depth: Final = int(kwargs.get("_agentic_loop_depth", 0) or 0)
-        max_loops: Final = int(kwargs.get("max_agentic_loops", 3) or 3)
+        configured: Final = validated_max_agentic_loops(
+            kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
+        )
+        max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
         fingerprints: Final = list(kwargs.get("_agentic_loop_fingerprints", []) or [])
-        return depth, max(max_loops, 1), fingerprints
+        return depth, max_loops, fingerprints
 
     @staticmethod
     def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool:
@@ -5120,7 +5130,8 @@ class BaseLLMHTTPHandler:
         """
         Evaluate agentic-loop safety guards (fingerprint cycle / max depth).
 
-        Raises ValueError on abort.  Returns the current fingerprint on success.
+        Raises AgenticLoopSafetyError on abort.  Returns the current fingerprint
+        on success.
 
         These checks must not be swallowed by the per-callback ``except Exception``
         block that wraps callback dispatch — they are bounded-loop / cycle-break
@@ -5128,9 +5139,9 @@ class BaseLLMHTTPHandler:
         """
         fingerprint: Final = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls)
         if fingerprint in fingerprints:
-            raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
+            raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
         if depth >= max_loops:
-            raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
+            raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
         return fingerprint
 
     @staticmethod
@@ -5140,6 +5151,97 @@ class BaseLLMHTTPHandler:
         except Exception:
             return str(tools)
 
+    @staticmethod
+    def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
+        """
+        Collect the ids and names of the tool calls a safety rail just refused.
+
+        Callbacks hand back either a bare list of tool calls or a dict wrapping
+        that list under ``tool_calls``, and both the anthropic and responses
+        shapes carry an ``id`` (or ``call_id``) plus a ``name``.
+        """
+        calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
+        if not isinstance(calls, list):
+            return frozenset(), frozenset()
+        dict_calls: Final = (call for call in calls if isinstance(call, dict))
+        fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
+        ids: Final = frozenset(
+            value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
+        )
+        names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
+        return ids, names
+
+    @staticmethod
+    def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
+        """
+        Whether this response block belongs to a tool call the rail refused.
+
+        An id settles it on its own, so a block carrying one is matched on the id
+        alone and a client's own tool call survives even where it happens to
+        share a name with a refused one. The name is only consulted for tool call
+        shapes that arrive without an id.
+        """
+        if not isinstance(block, dict) or block.get("type") != "tool_use":
+            return False
+        block_id: Final = block.get("id")
+        if isinstance(block_id, str) and refused_ids:
+            return block_id in refused_ids
+        return block.get("name") in refused_names
+
+    @staticmethod
+    def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
+        """
+        Whether a refused rerun can still be answered with a finalized turn.
+
+        Only the anthropic messages surface can. The responses surface carries a
+        pydantic model the finalizer does not rewrite, so it keeps raising, which
+        is what every surface did before this path learned to end the turn.
+
+        The messages and responses call sites pass ``stream=False``, because
+        interception converts an intercepted stream to non-streaming before the
+        loop runs and rebuilds the SSE stream from the finalized turn
+        afterwards. ``AgenticStreamingIterator`` passes ``stream=True``, and
+        that path keeps raising: its events are already on the wire, so a
+        finalized turn would reach the client as a second message rather than
+        as a replacement.
+        """
+        return not stream and api_surface == "anthropic_messages"
+
+    @staticmethod
+    def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
+        """
+        Turn the response into a terminal turn after a safety rail refused the rerun.
+
+        The refused tool calls target tools LiteLLM injected on the client's
+        behalf, so a client that never declared them cannot send back a matching
+        ``tool_result``. Their blocks are dropped and a ``tool_use`` stop reason
+        is closed out as ``end_turn``, which is what a provider-native web search
+        turn returns once it stops calling tools.
+
+        A ``tool_use`` block the client itself declared is left alone, and while
+        one is still in the response the stop reason stays ``tool_use`` so the
+        client knows to answer it.
+        """
+        if not isinstance(response, dict):
+            return response
+
+        refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
+        finalized: Final = dict(response)
+        content: Final = finalized.get("content")
+        if isinstance(content, list):
+            kept_blocks: Final = [
+                block
+                for block in content
+                if not BaseLLMHTTPHandler._is_refused_tool_use_block(block, refused_ids, refused_names)
+            ]
+            finalized["content"] = kept_blocks
+            client_tool_use_remains: Final = any(
+                isinstance(block, dict) and block.get("type") == "tool_use" for block in kept_blocks
+            )
+            if not client_tool_use_remains and finalized.get("stop_reason") == "tool_use":
+                finalized["stop_reason"] = "end_turn"
+        return finalized
+
     async def _execute_anthropic_agentic_plan(
         self,
         plan: AgenticLoopPlan,
@@ -5505,14 +5607,30 @@ class BaseLLMHTTPHandler:
                 continue
 
             # Safety guards must run OUTSIDE the callback try/except — they are
-            # bounded-loop / cycle-break rails that must propagate to the caller.
-            fingerprint = self._check_agentic_loop_safety(
-                tool_calls=tool_calls,
-                fingerprints=fingerprints,
-                depth=depth,
-                max_loops=max_loops,
-                model=model,
-            )
+            # bounded-loop / cycle-break rails, not callback bugs.
+            try:
+                fingerprint = self._check_agentic_loop_safety(
+                    tool_calls=tool_calls,
+                    fingerprints=fingerprints,
+                    depth=depth,
+                    max_loops=max_loops,
+                    model=model,
+                )
+            except AgenticLoopSafetyError as e:
+                if not self._can_replace_turn_with_terminal_response(stream, api_surface):
+                    raise
+                _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+                verbose_logger.warning(
+                    "LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
+                    _call_id,
+                    model,
+                    str(e),
+                )
+                return self._maybe_wrap_in_fake_stream(
+                    self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
+                    logging_obj,
+                    api_surface,
+                )
 
             try:
                 kwargs_with_provider = hook_kwargs.copy()

litellm/llms/fal_ai/cost_calculator.py

@@ -1,25 +1,75 @@
-from typing import Any, Final
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
 
 import litellm
 from litellm.types.utils import ImageResponse
 
+FAL_KEYED_PRICING_DEFAULT_QUALITY: Final[str] = "high"
+FAL_TEXT_TO_IMAGE_DEFAULT_SIZE: Final[str] = "1024-x-768"
+FAL_NAMED_IMAGE_SIZES: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "square_hd": "1024-x-1024",
+        "square": "512-x-512",
+        "portrait_4_3": "768-x-1024",
+        "portrait_16_9": "576-x-1024",
+        "landscape_4_3": "1024-x-768",
+        "landscape_16_9": "1024-x-576",
+    }
+)
+
+
+def _keyed_size(model: str, optional_params: Mapping[str, object]) -> str | None:
+    image_size: Final = optional_params.get("image_size")
+    if image_size is None:
+        return None if model.endswith("/edit") else FAL_TEXT_TO_IMAGE_DEFAULT_SIZE
+    if isinstance(image_size, Mapping):
+        width: Final = image_size.get("width")
+        height: Final = image_size.get("height")
+        if isinstance(width, int) and isinstance(height, int):
+            return f"{width}-x-{height}"
+        return None
+    if isinstance(image_size, str):
+        return FAL_NAMED_IMAGE_SIZES.get(image_size)
+    return None
+
+
+def _keyed_cost_per_image(model: str, optional_params: Mapping[str, object] | None) -> float | None:
+    if optional_params is None:
+        return None
+    size: Final = _keyed_size(model=model, optional_params=optional_params)
+    if size is None:
+        return None
+    raw_quality: Final = optional_params.get("quality")
+    quality: Final = (
+        raw_quality if isinstance(raw_quality, str) and raw_quality != "auto" else FAL_KEYED_PRICING_DEFAULT_QUALITY
+    )
+    keyed_entry: Final = litellm.model_cost.get(f"fal_ai/{quality}/{size}/{model}")
+    if keyed_entry is None:
+        return None
+    keyed_cost: Final = keyed_entry.get("output_cost_per_image")
+    return float(keyed_cost) if isinstance(keyed_cost, (int, float)) else None
+
 
 def cost_calculator(
     model: str,
-    image_response: Any,
+    image_response: object,
+    optional_params: Mapping[str, object] | None = None,
 ) -> float:
     """
     fal.ai image generation cost calculator
     """
+    if not isinstance(image_response, ImageResponse):
+        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    # the proxy cost path passes the provider-prefixed model name
+    model = model.removeprefix(f"{litellm.LlmProviders.FAL_AI.value}/")
+    num_images: Final[int] = len(image_response.data) if image_response.data else 0
+    keyed_cost_per_image: Final = _keyed_cost_per_image(model=model, optional_params=optional_params)
+    if keyed_cost_per_image is not None:
+        return keyed_cost_per_image * num_images
     _model_info: Final = litellm.get_model_info(
         model=model,
         custom_llm_provider=litellm.LlmProviders.FAL_AI.value,
     )
     output_cost_per_image: Final[float] = _model_info.get("output_cost_per_image") or 0.0
-    num_images: int = 0
-    if isinstance(image_response, ImageResponse):
-        if image_response.data:
-            num_images = len(image_response.data)
-        return output_cost_per_image * num_images
-    else:
-        raise ValueError(f"image_response must be of type ImageResponse got type={type(image_response)}")
+    return output_cost_per_image * num_images

litellm/llms/openai/openai.py

@@ -22,7 +22,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.files.types import FileContentStreamingResult
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.invoke_handler import MockResponseIterator
@@ -1365,9 +1365,21 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 client=client,
             )
 
-            if headers:
-                data["extra_headers"] = headers
-            response = await openai_aclient.images.generate(**data, timeout=timeout)
+            logging_obj.pre_call(
+                input=prompt,
+                api_key=openai_aclient.api_key,
+                additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                    "headers": {"Authorization": f"Bearer {openai_aclient.api_key}"},  # mutable-ok: logged header map
+                    "api_base": str(openai_aclient.base_url),
+                    "acompletion": True,
+                    "complete_input_dict": data,
+                },
+            )
+
+            request_data: Final = (  # mutable-ok: the OpenAI SDK takes the request body as a dict
+                {**data, "extra_headers": headers} if headers else data
+            )
+            response = await openai_aclient.images.generate(**request_data, timeout=timeout)
             stringified_response: Final = response.model_dump()
             ## LOGGING
             logging_obj.post_call(
@@ -1450,9 +1462,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             )
 
             ## COMPLETION CALL
-            if headers:
-                data["extra_headers"] = headers
-            _response: Final = openai_client.images.generate(**data, timeout=timeout)
+            request_data: Final = (  # mutable-ok: the OpenAI SDK takes the request body as a dict
+                {**data, "extra_headers": headers} if headers else data
+            )
+            _response: Final = openai_client.images.generate(**request_data, timeout=timeout)
 
             response: Final = _response.model_dump()
             ## LOGGING
@@ -1501,6 +1514,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         project: str | None,
         max_retries: int,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
         aspeech: bool | None = None,
         client=None,
         shared_session: Optional["ClientSession"] = None,
@@ -1517,6 +1531,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
                 project=project,
                 max_retries=max_retries,
                 timeout=timeout,
+                logging_obj=logging_obj,
                 client=client,
                 shared_session=shared_session,
             )
@@ -1531,7 +1546,17 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             shared_session=shared_session,
         )
 
-        response: Final = cast(OpenAI, openai_client).audio.speech.create(
+        sync_client: Final = cast(OpenAI, openai_client)
+        logging_obj.pre_call(
+            input=input,
+            api_key=api_key,
+            additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                "complete_input_dict": speech_request_body(model, voice, optional_params),
+                "api_base": str(sync_client.base_url),
+            },
+        )
+
+        response: Final = sync_client.audio.speech.create(
             model=model,
             voice=voice,
             input=input,
@@ -1551,6 +1576,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         project: str | None,
         max_retries: int,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
         client=None,
         shared_session: Optional["ClientSession"] = None,
     ) -> HttpxBinaryResponseContent:
@@ -1567,6 +1593,15 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
             ),
         )
 
+        logging_obj.pre_call(
+            input=input,
+            api_key=api_key,
+            additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                "complete_input_dict": speech_request_body(model, voice, optional_params),
+                "api_base": str(openai_client.base_url),
+            },
+        )
+
         response: Final = await openai_client.audio.speech.create(
             model=model,
             voice=voice,

litellm/llms/openai_like/providers.json

@@ -188,5 +188,17 @@
       "max_completion_tokens": "max_tokens"
     },
     "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
+  },
+  "scx-ai": {
+    "base_url": "https://api.scx.ai/v1",
+    "api_key_env": "SCX_API_KEY",
+    "api_base_env": "SCX_API_BASE",
+    "param_mappings": {
+      "max_completion_tokens": "max_tokens"
+    },
+    "constraints": {
+      "temperature_max": 1.99
+    },
+    "supported_endpoints": ["/v1/chat/completions"]
   }
 }

litellm/llms/sagemaker/chat/transformation.py

@@ -54,7 +54,30 @@ class SagemakerChatConfig(OpenAIGPTConfig, BaseAWSLLM):
         api_key: str | None = None,
         api_base: str | None = None,
     ) -> dict:
-        return headers
+        inference_component_name: Final = optional_params.get("model_id")
+        if not isinstance(inference_component_name, str):
+            return headers
+        return {**headers, "X-Amzn-SageMaker-Inference-Component": inference_component_name}
+
+    def transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: matches the base chat transform signature
+        optional_params: dict,  # mutable-ok: matches the base chat transform signature
+        litellm_params: dict,  # mutable-ok: matches the base chat transform signature
+        headers: dict,  # mutable-ok: matches the base chat transform signature
+    ) -> dict:  # mutable-ok: the handler sends this body straight to httpx
+        request: Final = super().transform_request(
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        served_model_name: Final = litellm_params.get("hf_model_name")
+        if not isinstance(served_model_name, str):
+            return request
+        return {**request, "model": served_model_name}
 
     def get_complete_url(
         self,

litellm/llms/vertex_ai/common_utils.py

@@ -1164,21 +1164,31 @@ class VertexAITokenCounter(BaseTokenCounter):
                     original_response=result,
                 )
         else:
-            # Use standard Vertex AI (Gemini) token counter
             from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
+            from litellm.llms.vertex_ai.gemini.transformation import (
+                _gemini_convert_messages_with_history,  # pyright: ignore[reportPrivateUsage]  # shared helper already used by gemini/chat, context_caching, and vertex_and_google_ai_studio_gemini
+            )
+
+            resolved_contents: Final = (
+                contents
+                if contents is not None
+                else _gemini_convert_messages_with_history(
+                    messages=messages or []  # mutable-ok: fallback for None messages; helper signature requires list
+                )
+            )
 
             count_tokens_params: Final = {
                 "model": model_to_use,
-                "contents": contents,
+                "contents": resolved_contents,
             }
             count_tokens_params_request.update(count_tokens_params)
             result = await VertexAITokenCounter().acount_tokens(
                 **count_tokens_params_request,
             )
 
-            if result is not None:
+            if result is not None and "totalTokens" in result:
                 return TokenCountResponse(
-                    total_tokens=result.get("totalTokens", 0),
+                    total_tokens=result["totalTokens"],
                     request_model=request_model,
                     model_used=model_to_use,
                     tokenizer_type=result.get("tokenizer_used", ""),

litellm/main.py

@@ -5091,14 +5091,16 @@ def completion(
     model_info: Final = kwargs.get("model_info", None)
     proxy_server_request: Final = kwargs.get("proxy_server_request", None)
     fallbacks = kwargs.get("fallbacks", None)
-    provider_specific_header: Final = cast(ProviderSpecificHeader | None, kwargs.get("provider_specific_header", None))
+    provider_specific_header: Final = cast(
+        ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
+        kwargs.get("provider_specific_header", None),
+    )
     headers = kwargs.get("headers", None) or extra_headers
 
     ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None)
     user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None)
     assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None)
-    if headers is None:
-        headers = {}
+    headers = {} if headers is None else dict(headers)
     if extra_headers is not None:
         headers.update(extra_headers)
     # Inject proxy auth headers if configured
@@ -7535,6 +7537,15 @@ async def amoderation(
             },
             custom_llm_provider=custom_llm_provider,
         )
+        moderation_request: Final = {"input": input, "model": model}  # mutable-ok: logged as the raw request body
+        litellm_logging_obj.pre_call(
+            input=input,
+            api_key=api_key,
+            additional_args={  # mutable-ok: loggers isinstance-check this payload as a dict
+                "complete_input_dict": moderation_request,
+                "api_base": str(_openai_client.base_url),
+            },
+        )
 
     if model is not None:
         response = await _openai_client.moderations.create(input=input, model=model)
@@ -8040,6 +8051,7 @@ def speech(
             project=project,
             max_retries=max_retries,
             timeout=timeout,
+            logging_obj=logging_obj,
             client=client,  # pass AsyncOpenAI, OpenAI client
             aspeech=aspeech,
             shared_session=shared_session,
@@ -8118,6 +8130,7 @@ def speech(
                 organization=organization,
                 max_retries=max_retries,
                 timeout=timeout,
+                logging_obj=logging_obj,
                 client=client,  # pass AsyncOpenAI, OpenAI client
                 aspeech=aspeech,
                 litellm_params=litellm_params_dict,

litellm/provider_endpoints_support_backup.json

@@ -2027,6 +2027,23 @@
         "interactions": true
       }
     },
+    "scx-ai": {
+      "display_name": "SCX.ai (`scx-ai`)",
+      "url": "https://docs.litellm.ai/docs/providers/scx_ai",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "snowflake": {
       "display_name": "Snowflake (`snowflake`)",
       "url": "https://docs.litellm.ai/docs/providers/snowflake",

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -39,6 +39,7 @@ from litellm.proxy._types import (
     SpecialMCPServerName,
     SpecialMCPServerNames,
     UserAPIKeyAuth,
+    user_api_key_has_admin_view,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.auth.user_api_key_auth import (
@@ -160,7 +161,7 @@ def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> b
     """True when this auth is a keyless subject admitted by the gateway session / bridge user
     path, as opposed to a JWT or other keyless auth that merely lacks a ``team_id``.
 
-    Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``_reload_admitted_user``. It
+    Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``reload_admitted_user``. It
     is deliberately NOT a ``metadata`` key, which is caller-controlled at key creation and so forgeable
     on a personal key to gain the team grant union or dodge the egress scrub; this field cannot be."""
     return user_api_key_auth is not None and user_api_key_auth.mcp_admitted_user_subject is True
@@ -812,7 +813,7 @@ class MCPRequestHandler:
 
         Identity-only sibling of :meth:`_admit_dcr_bridge_delegate`: the session token seals no
         upstream credential (those are vaulted per user, resolved at egress), so authorization is
-        resolved fresh via :meth:`_reload_admitted_user` + the centralized policy gate rather than a
+        resolved fresh via :meth:`reload_admitted_user` + the centralized policy gate rather than a
         mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard
         pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired,
         tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user."""
@@ -835,7 +836,7 @@ class MCPRequestHandler:
         match result:
             case SessionBearerAdmitted():
                 try:
-                    admitted: Final = await MCPRequestHandler._reload_admitted_user(result.principal.user_id)
+                    admitted: Final = await MCPRequestHandler.reload_admitted_user(result.principal.user_id)
                     admitted.mcp_session_resource_server_id = result.principal.resource_server_id
                     await MCPRequestHandler._enforce_admitted_live_policy(
                         admitted=admitted, request=request, route=route
@@ -893,12 +894,12 @@ class MCPRequestHandler:
             case "key_hash":
                 return await MCPRequestHandler._reload_admitted_key(identity.subject)
             case "user_id":
-                return await MCPRequestHandler._reload_admitted_user(identity.subject)
+                return await MCPRequestHandler.reload_admitted_user(identity.subject)
             case _:
                 assert_never(identity.subject_type)
 
     @staticmethod
-    async def _reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
+    async def reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
         """Reload the live user an interactively-minted envelope references and admit them as themselves.
 
         The user's own object permission and ``org_id`` ride on the returned ``UserAPIKeyAuth``, and the
@@ -1785,11 +1786,14 @@ class MCPRequestHandler:
             global_mcp_server_manager,
         )
 
-        # An OPEN channel (allow_all_keys, the user's own BYOM) makes the server REACHABLE through the
-        # user, though no grant source names it — without this the union returns [], listable but
-        # uninvokable. Reachability is ALL it confers, NOT a ceiling waiver: the user's own
-        # mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do on an allow_all server.
-        reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(auth)
+        # An OPEN channel (allow_all_keys, the user's own BYOM, an unscoped admin-view role) makes the
+        # server REACHABLE through the user, though no grant source names it — without this the union
+        # returns [], listable but uninvokable. Reachability is ALL it confers, NOT a ceiling waiver:
+        # the user's own mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do
+        # on an allow_all server or an admin key's do on any server.
+        reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(
+            auth
+        ) or await MCPRequestHandler.admin_view_unscoped(auth)
 
         allowed: Final[set[str]] = set()
         for source, granted in await MCPRequestHandler.admitted_source_grants(auth):
@@ -2723,6 +2727,32 @@ class MCPRequestHandler:
         entitled_servers: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_user(user_api_key_auth)
         return entitled_servers is None or len(entitled_servers) > 0
 
+    @staticmethod
+    async def admin_view_unscoped(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool:
+        """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
+        credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
+
+        Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
+        ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
+        admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
+        [] by DB default, so for that shape the row binds through the entitlement ceiling instead
+        (any non-empty entitlement, or an unresolved one, disqualifies), exactly as
+        ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
+        server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
+        channel in ``_resolve_admitted_subject_tools`` both consult it, so the two axes cannot
+        disagree."""
+        if user_api_key_auth is None or not user_api_key_has_admin_view(user_api_key_auth):
+            return False
+        object_permission: Final = user_api_key_auth.object_permission
+        credential_scoped: Final = (
+            not _is_mcp_admitted_user_subject(user_api_key_auth)
+            and object_permission is not None
+            and object_permission.mcp_servers is not None
+        )
+        if credential_scoped:
+            return False
+        return not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
+
     @staticmethod
     async def _apply_user_tool_ceiling(
         allowed_tools: Sequence[str] | None,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -750,6 +750,55 @@ def _redirect_to_upstream_authorize(
     return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params))))
 
 
+def _bridge_access_denied_redirect(redirect_uri: str, state: str, mcp_server: MCPServer) -> RedirectResponse:
+    """RFC 6749 section 4.1.2.1 denial for the interactive bridge authorize, delivered to the
+    already-validated client redirect_uri so a DCR client surfaces the failure at connect time."""
+    server_label: Final = mcp_server.alias or mcp_server.server_name or mcp_server.server_id
+    params: Final = {
+        "error": "access_denied",
+        "error_description": (
+            f"the signed-in user has no access to MCP server '{server_label}' on this gateway; "
+            "grant it through a team or user object permission, or mark the server allow_all_keys"
+        ),
+        **({"state": state} if state else {}),
+    }
+    return RedirectResponse(_append_query_params(redirect_uri, params), status_code=302)
+
+
+async def _bridge_authorize_access_denial(
+    litellm_user_id: str,
+    mcp_server: MCPServer,
+    redirect_uri: str,
+    state: str,
+) -> RedirectResponse | None:
+    """The denial redirect for a signed-in user who cannot reach the target server, or None to proceed.
+
+    Admits the user exactly as MCP egress will (the same ``reload_admitted_user`` constructor and the
+    same ``get_allowed_mcp_servers`` resolver), so an envelope is minted only when the resulting
+    session can actually list and call the server's tools. Without this gate the flow completes, the
+    client shows connected, and every tool request fail-closes to an empty list with nothing telling
+    the operator why. An availability fault (5xx, e.g. a DB outage's 503) propagates; an unknown or
+    deactivated user denies like a missing grant, fail closed.
+    """
+    from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
+        MCPRequestHandler,
+    )
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+        global_mcp_server_manager,
+    )
+
+    try:
+        admitted: Final = await MCPRequestHandler.reload_admitted_user(litellm_user_id)
+    except HTTPException as exc:
+        if exc.status_code >= 500:
+            raise
+        return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
+    allowed_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(admitted)
+    if mcp_server.server_id in allowed_server_ids:
+        return None
+    return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
+
+
 async def authorize_with_server(
     request: Request,
     mcp_server: MCPServer,
@@ -819,6 +868,14 @@ async def authorize_with_server(
         litellm_user_id = _user_id_from_session_cookie(request)
         if litellm_user_id is None:
             return _redirect_to_litellm_login(request)
+        denial: Final = await _bridge_authorize_access_denial(
+            litellm_user_id=litellm_user_id,
+            mcp_server=mcp_server,
+            redirect_uri=redirect_uri,
+            state=state,
+        )
+        if denial is not None:
+            return denial
 
     encoded_state: Final = encode_state_with_base_url(
         base_url=base_url,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -2943,17 +2943,14 @@ class MCPServerManager:
         2. If admin and no object_permission, return all servers
         3. Otherwise, use standard permission checks
         """
-        from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
-
         allow_all_server_ids: Final = self.get_allow_all_keys_server_ids()
 
         # A keyless admitted subject is resolved per grant source, and channel decisions that are
         # absolute for a scoped KEY credential are not absolute for it: its own opt-out silences its
-        # own source (handled per source in the resolver), never its teams' grants, and its admin
-        # role does not swallow the grant model — a session bearer is a third-party client
-        # credential, not the dashboard, so an admin signing in through the connect flow gets their
-        # grants like anyone else rather than handing the client the full registry ahead of every
-        # per-team org ceiling.
+        # own source (handled per source in the resolver), never its teams' grants. Its admin role
+        # rides the HUMAN, not the credential: an admin's session resolves the same registry their
+        # dashboard shows (connect-page parity), bounded like an admin key by explicit
+        # object_permission scope, the entitlement ceiling, and the session resource scope below.
         is_admitted_subject: Final = _is_mcp_admitted_user_subject(user_api_key_auth)
 
         # The key explicitly opted out of every MCP server. Return zero before
@@ -2982,26 +2979,16 @@ class MCPServerManager:
         )
 
         try:
-            # If admin but NO explicit object permission, get all servers (never for an admitted
-            # subject — see is_admitted_subject above)
-            if (
-                user_api_key_auth
-                and not is_admitted_subject
-                and _user_has_admin_view(user_api_key_auth)
-                and not has_explicit_object_permission
-                # An entitlement attached to the HUMAN binds them whatever their role: it is the
-                # person's scope, not the credential's, so an admin role is not a waiver of it. An
-                # UNRESOLVED entitlement also skips the shortcut, so the resolver denies rather than
-                # handing over the whole registry on a transient fault.
-                and not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
-            ):
-                verbose_logger.debug("Admin user without explicit object_permission - returning all servers")
-                return list(self.get_registry().keys())
-
-            # Get allowed servers from object permissions (respects object_permission even for admins)
-            allowed_mcp_servers: Final = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth)
-            verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers)
-            combined_servers: Final = set(allowed_mcp_servers)
+            # Admin view with no explicit object permission and no entitlement ceiling resolves the
+            # whole registry, for keys AND admitted session subjects alike (one predicate owns the
+            # question). Seeded into the union rather than returned early so the session resource
+            # scope below still bounds a per-server envelope held by an admin.
+            combined_servers: Final = (
+                set(self.get_registry().keys())
+                if await MCPRequestHandler.admin_view_unscoped(user_api_key_auth)
+                else set(await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth))
+            )
+            verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", combined_servers)
             combined_servers.update(
                 await self.operator_open_server_ids(
                     user_api_key_auth,

litellm/proxy/_experimental/mcp_server/oauth_utils.py

@@ -180,6 +180,22 @@ def well_known_root_suffix() -> str:
     return "" if root == "/" else root
 
 
+def get_route_relative_request_path(scope: Scope) -> str:
+    """The request path the MCP route shapes are written against: the raw ASGI path with the
+    deployment's ``root_path`` removed.
+
+    ``scope["path"]`` and ``_original_path`` are both raw request-line paths, so on a sub-path
+    deployment they still carry the ``SERVER_ROOT_PATH`` prefix (``/litellm/{server}/mcp``) while
+    every route shape compared against them is root-relative. Mirrors the segment-boundary strip in
+    :func:`litellm.proxy.auth.auth_utils.get_request_route`, which the rest of the MCP auth path
+    already routes through, so ``/litellmfoo`` is not truncated under ``root_path=/litellm``."""
+    raw_path = str(scope.get("_original_path") or scope.get("path", "") or "")
+    root_path = str(scope.get("app_root_path") or scope.get("root_path") or "").rstrip("/")
+    if root_path and (raw_path == root_path or raw_path.startswith(f"{root_path}/")):
+        return raw_path[len(root_path) :]
+    return raw_path
+
+
 def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str:
     """The per-server protected-resource metadata URL matching the spelling the request
     arrived on, so a strict RFC 9728 client resolves the same route the proxy registered.
@@ -188,7 +204,7 @@ def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str
     the route decorators insert it (see :func:`well_known_root_suffix`)."""
     request: Final = Request(scope)
     base_url: Final = get_request_base_url(request)
-    _path: Final = scope.get("_original_path") or scope.get("path", "") or ""
+    _path: Final = get_route_relative_request_path(scope)
 
     if _path.startswith(f"/{server_name}/mcp"):
         return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp"

litellm/proxy/_experimental/mcp_server/server.py

@@ -51,6 +51,8 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     _redact_mcp_resource_url,
     get_passthrough_www_authenticate,
+    get_route_relative_request_path,
+    well_known_root_suffix,
 )
 from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_DESCRIPTION,
@@ -3782,14 +3784,15 @@ if MCP_AVAILABLE:
 
                     request = StarletteRequest(scope)
                     base_url = get_request_base_url(request)
-                    _path = scope.get("_original_path") or scope.get("path", "") or ""
+                    _path = get_route_relative_request_path(scope)
 
                     # Pick the well-known AS-metadata form that matches the inbound route
                     # so strict RFC 9728 §3.2 clients can resolve it correctly.
+                    as_metadata_root = f"{base_url}/.well-known/oauth-authorization-server{well_known_root_suffix()}"
                     if _path.startswith(f"/mcp/{server_name}"):
-                        _as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}"
+                        _as_url = f"{as_metadata_root}/mcp/{server_name}"
                     else:
-                        _as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}"
+                        _as_url = f"{as_metadata_root}/{server_name}"
                     authorization_uri = f'Bearer authorization_uri="{_as_url}"'
 
                     raise HTTPException(

litellm/proxy/_experimental/mcp_server/ui_session_utils.py

@@ -91,7 +91,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey
     )
 
     try:
-        admitted: Final = await MCPRequestHandler._reload_admitted_user(user_id)
+        admitted: Final = await MCPRequestHandler.reload_admitted_user(user_id)
     except HTTPException as e:
         verbose_logger.warning("MCP dashboard session: admitted-subject reload failed for %s: %s", user_id, e.detail)
         return None

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.__PAGE__.txt

@@ -1,9 +1,9 @@
 1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ClientPageRoot"]
-3:I[871135,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js","/litellm-asset-prefix/_next/static/chunks/3l0glczkblv8_.js","/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","/litellm-asset-prefix/_next/static/chunks/2ca0bgyj3-r_j.js","/litellm-asset-prefix/_next/static/chunks/29nmr1sywlx25.js","/litellm-asset-prefix/_next/static/chunks/0gh1eppc9ekzh.js","/litellm-asset-prefix/_next/static/chunks/3hk5c4q5k-j7x.js","/litellm-asset-prefix/_next/static/chunks/2mhbxmykyh83f.js","/litellm-asset-prefix/_next/static/chunks/1xk5l9lxa0dv-.js","/litellm-asset-prefix/_next/static/chunks/26e7zpdybuhtq.js","/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","/litellm-asset-prefix/_next/static/chunks/0m-cn894wctv5.js","/litellm-asset-prefix/_next/static/chunks/3cw_k7_vr9pcu.js","/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","/litellm-asset-prefix/_next/static/chunks/2udc_95331vyv.js","/litellm-asset-prefix/_next/static/chunks/3mkd81u36rwju.js","/litellm-asset-prefix/_next/static/chunks/0kh9ov64og3-k.js","/litellm-asset-prefix/_next/static/chunks/3580ki1m5g-sx.js","/litellm-asset-prefix/_next/static/chunks/2xuwoxcnxuv39.js","/litellm-asset-prefix/_next/static/chunks/3bwziv83xzehe.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"OutletBoundary"]
+2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
+3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js"],"default"]
+6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
 7:"$Sreact.suspense"
-0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3l0glczkblv8_.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1cea03gg5a_c7.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2ca0bgyj3-r_j.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/29nmr1sywlx25.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0gh1eppc9ekzh.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/3hk5c4q5k-j7x.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2mhbxmykyh83f.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1xk5l9lxa0dv-.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/26e7zpdybuhtq.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2c90xukbd3il6.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0m-cn894wctv5.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/3cw_k7_vr9pcu.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/12wsfsljxg4xv.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/2udc_95331vyv.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/3mkd81u36rwju.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/0kh9ov64og3-k.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/3580ki1m5g-sx.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/2xuwoxcnxuv39.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/3bwziv83xzehe.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
+0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1e-4-g6x6zyse.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/0gygfcpmiijl8.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2cu4j3g1tldv4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2kmqjpt047tjo.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1phty1k2nx8fx.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0u3cfuz-tf0wj.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/39s4-rh6l9sa1.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/3a3jpg95umjho.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/28hnu_qv5e_c_.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0qf1_0kt4uuxa.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3kpec-qy1uzod.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
 4:{}
 5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
 8:null

litellm/proxy/_experimental/out/__next.!KGRhc2hib2FyZCk.txt

@@ -1,7 +1,7 @@
 1:"$Sreact.fragment"
-2:I[92825,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ClientSegmentRoot"]
-3:I[216370,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
-0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/21b4hw_igldhz.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2_hxghav3pe9j.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/0g_w4tf2inv3i.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1gwzs-8xkvx8f.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/1oob52g5gib5j.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1yok3x3_3gr1p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2mxcro_n8i1ef.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/3xciut9pzmr7-.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/1jkcw8ug0uobj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/3k3r6waxmnsvu.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
+2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
+3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js"],"default"]
+4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/37t2cfzl_b58p.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/1zf358k334atp.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3kil-7y33kpm9.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/1natmx9lu3mus.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1dmg55q8kht9j.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1lrl_8p0h2sbm.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}
 6:"$0:rsc:props:children:1:props:serverProvidedParams:params"

litellm/proxy/_experimental/out/__next._head.txt

@@ -1,6 +1,6 @@
 1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"MetadataBoundary"]
+2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
+3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
 4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"IconMark"]
-0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
+5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
+0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

litellm/proxy/_experimental/out/__next._index.txt

@@ -1,10 +1,11 @@
 1:"$Sreact.fragment"
-2:I[12985,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"NuqsAdapter"]
-3:I[867271,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
-4:I[71195,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
-5:I[557951,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"AuthProvider"]
-6:I[339756,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
-7:I[837457,["/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js"],"default"]
+2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
+3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
+4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
+6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
+8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","style"]
-0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/19frz_r2jewoi.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3_zdkdwptdu3w.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/1dg0y22lcfxz2.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"TeJ852IBdcKgsOMzGKY73"}
+:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
+0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3_06chgeyldml.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/26h-ny89yaww0.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"0cE25rDXvGu3tj4HWOGKy"}

litellm/proxy/_experimental/out/__next._tree.txt

@@ -1,4 +1,4 @@
 :HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/0cefehsj9nby1.css","style"]
+:HL["/litellm-asset-prefix/_next/static/chunks/2hicgq-mjp8vy.css","style"]
 :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"TeJ852IBdcKgsOMzGKY73"}
+0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"0cE25rDXvGu3tj4HWOGKy"}