Merge branch 'litellm_internal_staging' of https://github.com/BerriAI/litellm into litellm_team_rate_limit_prometheus_metrics

.github/CODEOWNERS

@@ -1,5 +1,6 @@
-/ui/ @yuneng-jiang @ryan-crabbe-berri
-/litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/ @yuneng-berri @ryan-crabbe-berri
+/litellm/proxy/_experimental/out/ @yuneng-berri @ryan-crabbe-berri
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
 /model_prices_and_context_window.json @mateo-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri
+/litellm-proxy-extras/litellm_proxy_extras/migrations/ @yuneng-berri @ryan-crabbe-berri

.github/actions/cache-cargo-build/action.yml

@@ -0,0 +1,31 @@
+name: "Cache the Rust build"
+description: >-
+  Cache the Cargo registry and target directory the root package's build needs,
+  so only the first job on a given Cargo.lock compiles the bridge from scratch.
+
+  litellm builds through maturin, which compiles litellm-rust/crates/python-bridge
+  in release mode before it can produce a wheel. `uv sync` therefore pays a full
+  build in every job that installs the workspace: measured at 2m40s per unit shard
+  on 2026-08-21, more than the whole unit tier spends running tests. Nothing caught
+  it, because the uv cache holds wheels uv downloads rather than wheels it builds,
+  and a path dependency whose source moves every commit could never hit that cache
+  anyway. Cargo rebuilds only what changed when its target directory survives, so a
+  warm job pays for the bridge crate alone.
+
+  The key namespace is separate from test-rust.yml's. Both cache the same directory,
+  but that workflow fills it with debug and clippy artifacts, which a release build
+  cannot reuse, and a shared key would let whichever ran first deny the other a save.
+
+runs:
+  using: composite
+  steps:
+    - name: Restore the Cargo registry and target directory
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        path: |
+          ~/.cargo/registry
+          ~/.cargo/git
+          litellm-rust/target
+        key: ${{ runner.os }}-cargo-release-${{ hashFiles('litellm-rust/Cargo.lock') }}
+        restore-keys: |
+          ${{ runner.os }}-cargo-release-

.github/ci-coverage-allowlist.yml

@@ -48,16 +48,6 @@ test_paths:
       choice it informed is settled
     paths:
       - tests/code_coverage_tests/test_aio_http_image_conversion.py
-  - reason: >-
-      The last file of a second mirror that sat beside tests/test_litellm and ran nowhere. Its
-      other 33 files landed in the real mirror during August 2026, 30 as moves and 3 by merging
-      their bodies into the live file of the same name. This one cannot follow either route yet:
-      its live twin was rewritten from 1268 lines to 9434, and of the 19 tests here 5 have no
-      counterpart while 25 assertions fail against today's code, so what survives that rewrite
-      is a judgement about the endpoints, not a merge. Revisit by deciding which of the five
-      behaviours still hold
-    paths:
-      - tests/litellm/proxy/_experimental/mcp_server/test_discoverable_endpoints.py
   - reason: >-
       No job invokes this suite and its files mix pure transformation tests with ones driving live
       vendor vector stores, so assigning them needs a per-file decision

.github/workflows/_test-unit-base.yml

@@ -27,7 +27,7 @@ on:
         default: 20
       job-timeout-minutes:
         description: >-
-          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 40: 35 for
           the per-step ceilings on the setup steps below, and 5 for the runner
           overhead the job clock charges but no step owns (job init, step
           transitions, post-job cleanup). That headroom is what makes the test
@@ -36,7 +36,7 @@ on:
           arithmetic, so the sum is passed in rather than computed.
         required: false
         type: number
-        default: 55
+        default: 60
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -103,6 +103,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 5
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8

.github/workflows/check-ui-api-types.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install backend dependencies
         if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

.github/workflows/mutation-test.yml

@@ -53,6 +53,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml

.github/workflows/publish-basedpyright-base-counts.yml

@@ -43,6 +43,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Cache Prisma binaries
         uses: ./.github/actions/cache-prisma-binaries
 

.github/workflows/test-code-quality.yml

@@ -56,6 +56,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: uv sync --frozen --all-groups --all-extras
 

.github/workflows/test-linting.yml

@@ -78,6 +78,10 @@ jobs:
         run: |
           uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-mcp.yml

@@ -47,6 +47,10 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-terraform-provider.yml

@@ -88,6 +88,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router

.github/workflows/test-unit-documentation.yml

@@ -67,6 +67,10 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-uv-
 
+      - name: Cache the Rust build
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |

.github/workflows/test-unit.yml

@@ -55,7 +55,7 @@ jobs:
             workers: 2
             reruns: 1
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: enterprise-routing
             artifact-name: enterprise-routing
@@ -67,7 +67,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: integrations
             artifact-name: integrations
@@ -75,7 +75,7 @@ jobs:
             workers: 2
             reruns: 3
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: Vertex AI
             artifact-name: llm-vertex-ai
@@ -83,7 +83,7 @@ jobs:
             workers: 1
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: All Other Providers
             artifact-name: llm-other-providers
@@ -91,7 +91,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: misc
             artifact-name: misc
@@ -122,7 +122,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-auth
             artifact-name: proxy-auth
@@ -134,7 +134,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-endpoints
             artifact-name: proxy-endpoints
@@ -171,7 +171,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-server
             artifact-name: proxy-server
@@ -179,7 +179,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 60
-            job-timeout-minutes: 95
+            job-timeout-minutes: 100
 
           - shard: proxy-infra
             artifact-name: proxy-infra
@@ -198,7 +198,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types
@@ -209,7 +209,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
     uses: ./.github/workflows/_test-unit-base.yml
     with:
       test-path: ${{ matrix.test-path }}

.github/workflows/weekly_load_anomaly.yml

@@ -47,6 +47,9 @@ jobs:
         with:
           version: "0.10.9"
 
+      - name: Cache the Rust build
+        uses: ./.github/actions/cache-cargo-build
+
       - name: Install dependencies
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy

basedpyright-code-budget.json

@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1823
+    "limit": 1822
   },
   "reportRedeclaration": {
     "limit": 8

ci_cd/generate_model_prices_schema.py

@@ -27,6 +27,7 @@ EXTRA_BOOLEAN_KEYS = frozenset(
         "uses_embed_content",
         "use_openai_responses_path",
         "bedrock_converse_supports_strict_tools",
+        "thinking_always_on",
     }
 )
 

enterprise/enterprise_hooks/banned_keywords.py

@@ -21,6 +21,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BannedKeywords(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self):
         banned_keywords_list = litellm.banned_keywords_list

enterprise/enterprise_hooks/blocked_user_list.py

@@ -18,6 +18,7 @@ from fastapi import HTTPException
 
 
 class _ENTERPRISE_BlockedUserList(CustomLogger):
+    enforces_request_content: bool = True
     # Class variables or attributes
     def __init__(self, prisma_client: Optional[PrismaClient]):
         self.prisma_client = prisma_client

litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql

@@ -1,4 +0,0 @@
-UPDATE "LiteLLM_SpendLogs"
-SET "created_at" = "endTime",
-    "updated_at" = "endTime"
-WHERE "created_at" > "endTime" + interval '1 hour';

litellm/_logging.py

@@ -88,6 +88,24 @@ def redact_secrets(value: str) -> str:
     return _redact_string(value)
 
 
+def _substituted_color_message(record: logging.LogRecord) -> str | None:
+    """Render a record's ``color_message`` against its args, or None if absent.
+
+    uvicorn's colorized formatter re-renders `color_message` against
+    record.args at emit time (see uvicorn.logging.ColourizedFormatter) instead
+    of using the already-formatted record.msg, so it has to be substituted
+    before args are cleared or it is later formatted with no args and prints
+    the raw "%s://%s:%d" placeholders instead of the URL.
+    """
+    color_message: Final = record.__dict__.get("color_message")
+    if not isinstance(color_message, str) or not record.args:
+        return None
+    try:
+        return color_message % record.args
+    except TypeError:
+        return color_message
+
+
 class SecretRedactionFilter(logging.Filter):
     """Scrubs known secret/credential patterns from log records."""
 
@@ -97,6 +115,12 @@ class SecretRedactionFilter(logging.Filter):
         if not _ENABLE_SECRET_REDACTION:
             return True
 
+        # Runs before args are cleared, and before the extra-field loop below
+        # that redacts the substituted result.
+        substituted_color_message: Final = _substituted_color_message(record)
+        if substituted_color_message is not None:
+            record.color_message = substituted_color_message  # rebind-ok: a Filter scrubs records in place
+
         try:
             record.msg = _redact_string(record.getMessage())
             record.args = None

litellm/_redis.py

@@ -665,8 +665,16 @@ def get_redis_async_client(
         cluster_kwargs.setdefault("health_check_interval", REDIS_CLUSTER_HEALTH_CHECK_INTERVAL)
         cluster_kwargs.setdefault("socket_keepalive", True)
 
+        # A single node's client-side timeout must reset only that node's connections,
+        # not tear down the whole cluster client for every concurrent caller.
+        from litellm.caching.redis_cluster_node_isolation import (
+            get_litellm_async_redis_cluster_class,
+        )
+
+        async_redis_cluster_class: Final = get_litellm_async_redis_cluster_class()
+
         # Create async RedisCluster with IAM token as password if available
-        cluster_client: Final = async_redis.RedisCluster(
+        cluster_client: Final = async_redis_cluster_class(
             startup_nodes=new_startup_nodes,
             **cluster_kwargs,
         )

litellm/batches/batch_utils.py

@@ -296,6 +296,32 @@ def calculate_vertex_ai_batch_cost_and_usage(
     )
 
 
+def _provider_output_file_id(output_file_id: str) -> str:
+    """
+    Resolve the file id the provider actually knows: unified ids yield their embedded
+    llm_output_file_id, model-encoded ids decode to the raw provider id, raw ids pass through.
+    """
+    from litellm.proxy.openai_files_endpoints.common_utils import (
+        _is_base64_encoded_unified_file_id,
+        get_original_file_id,
+    )
+
+    unified_file_id: Final = _is_base64_encoded_unified_file_id(output_file_id)
+    if not unified_file_id:
+        return get_original_file_id(output_file_id)
+    try:
+        extracted: Final = unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
+    except (IndexError, AttributeError) as e:
+        verbose_logger.error(
+            "Failed to extract LLM output file ID from unified file ID: %s, error: %s",
+            output_file_id,
+            e,
+        )
+        return output_file_id
+    verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", extracted)
+    return extracted
+
+
 async def _fetch_batch_output_file_content(
     batch: Batch,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
@@ -311,23 +337,11 @@ async def _fetch_batch_output_file_content(
                        Required for Azure and other providers that need authentication
     """
     from litellm.files.main import afile_content
-    from litellm.proxy.openai_files_endpoints.common_utils import (
-        _is_base64_encoded_unified_file_id,
-    )
 
     if batch.output_file_id is None:
         raise ValueError("Output file id is None cannot retrieve file content")
 
-    file_id = batch.output_file_id
-    is_base64_unified_file_id: Final = _is_base64_encoded_unified_file_id(file_id)
-    if is_base64_unified_file_id:
-        try:
-            file_id = is_base64_unified_file_id.split("llm_output_file_id,")[1].split(";")[0]
-            verbose_logger.debug("Extracted LLM output file ID from unified file ID: %s", file_id)
-        except (IndexError, AttributeError) as e:
-            verbose_logger.error(
-                "Failed to extract LLM output file ID from unified file ID: %s, error: %s", batch.output_file_id, e
-            )
+    file_id: Final = _provider_output_file_id(batch.output_file_id)
 
     # Build kwargs for afile_content with credentials from litellm_params
     file_content_kwargs: Final = {

litellm/caching/redis_cluster_node_isolation.py

@@ -0,0 +1,173 @@
+"""Bounds the blast radius of a single node's transient connection error on the async
+Redis Cluster client.
+
+redis-py's ``RedisCluster._execute_command`` responds to a ``ConnectionError`` or
+``TimeoutError`` on ANY one node by tearing down every node's connections and flipping
+the client into "needs reinitialization", which forces every other concurrent caller
+sharing this client through one reinit lock until the whole cluster topology is
+re-walked. Under real proxy load, a client-side socket timeout on a single node is a
+routine event (the event loop was too busy to read the response before ``socket_timeout``
+elapsed) and does not mean the cluster's topology moved, so treating it as a full-cluster
+event turns one slow node into a proxy-wide latency spike while Redis itself stays
+healthy -- confirmed live: pausing one of three local cluster nodes made every concurrent
+command against the other two, untouched nodes stall for the full pause duration too.
+
+``get_litellm_async_redis_cluster_class`` returns a ``RedisCluster`` subclass that resets
+only the node that actually failed (mirroring what a plain, non-cluster Redis client
+already does when one of its pooled connections errors), leaving every other node's
+connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
+retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
+topology changed.
+"""
+
+import asyncio
+from typing import TYPE_CHECKING, Final, Protocol
+
+from litellm._logging import verbose_logger
+
+if TYPE_CHECKING:
+    from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
+
+
+class _ClusterNodeAttrs(Protocol):
+    """The subset of ``redis.asyncio.cluster.ClusterNode`` this override reads. redis-py
+    ships no resolvable stub for these members under the repo's current types-redis pin,
+    so a plain attribute access resolves every downstream use to ``Unknown`` under strict
+    mode; typing ``target_node`` as this Protocol at the one boundary keeps the override's
+    own logic fully typed without a banned ``typing.cast``."""
+
+    async def execute_command(
+        self,
+        *args: object,
+        **kwargs: object,  # kwargs-ok: mirrors redis-py's own ClusterNode.execute_command signature, a raw command dispatch with no fixed keyword contract
+    ) -> object: ...
+    async def disconnect(self) -> None: ...
+
+
+class _NodesManagerAttrs(Protocol):
+    _moved_exception: object
+
+    def get_node_from_slot(
+        self, slot: int, read_from_replicas: bool, load_balancing_strategy: object
+    ) -> _ClusterNodeAttrs: ...
+
+
+class _ClusterAttrs(Protocol):
+    RedisClusterRequestTTL: int
+    reinitialize_counter: int
+    reinitialize_steps: int
+    read_from_replicas: bool
+    load_balancing_strategy: object
+    nodes_manager: _NodesManagerAttrs
+
+    def get_node(self, node_name: str) -> _ClusterNodeAttrs: ...
+    async def _determine_slot(self, *args: object) -> int: ...
+    async def aclose(self) -> None: ...
+
+
+#: redis-py versions this override's copied ``_execute_command`` body has been verified
+#: against. A version outside this set may have changed the method's structure in a way
+#: this override can't see (Python won't error -- it'll just run our now-stale copy), so
+#: construction logs a loud warning rather than silently trusting an unverified copy.
+_VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
+
+
+def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
+    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+
+    Imported lazily because this module is reachable from a base ``import litellm`` while
+    redis is not a base dependency. Cheap to call repeatedly: the underlying redis
+    submodules are cached in ``sys.modules`` after the first import.
+    """
+    import redis
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.cluster import get_node_name
+    from redis.commands import READ_COMMANDS
+    from redis.exceptions import (
+        AskError,
+        BusyLoadingError,
+        ClusterDownError,
+        ClusterError,
+        MaxConnectionsError,
+        MovedError,
+        SlotNotCoveredError,
+        TryAgainError,
+    )
+    from redis.exceptions import ConnectionError as _RedisConnectionError
+    from redis.exceptions import TimeoutError as _RedisTimeoutError
+
+    if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
+        verbose_logger.warning(
+            "redis-py %s is not in the set this cluster-teardown-storm fix was verified "
+            "against (%s). The per-node-isolation override may not match the installed library's "
+            "real _execute_command behavior.",
+            redis.__version__,
+            sorted(_VERIFIED_REDIS_VERSIONS),
+        )
+
+    class LiteLLMAsyncRedisCluster(
+        _BaseAsyncRedisCluster  # pyright: ignore[reportUntypedBaseClass]  # same stale-stub gap as the import above; the base class itself is unresolvable, not this subclass's own code
+    ):
+        async def _execute_command(
+            self,
+            target_node: _ClusterNodeAttrs,
+            *args: object,
+            **kwargs: object,  # kwargs-ok: overrides redis-py's own **kwargs signature; the keyword contract is defined by the Redis command being dispatched, not by this method
+        ) -> object:
+            cluster: _ClusterAttrs = self
+            node = target_node
+
+            asking = moved = False
+            redirect_addr: str | None = None
+            ttl = cluster.RedisClusterRequestTTL
+
+            while ttl > 0:
+                ttl -= 1
+                try:
+                    if asking:
+                        assert redirect_addr is not None
+                        node = cluster.get_node(node_name=redirect_addr)
+                        await node.execute_command("ASKING")
+                        asking = False
+                    elif moved:
+                        slot = await cluster._determine_slot(*args)  # pyright: ignore[reportPrivateUsage]  # mirrors upstream's own un-overridden branch, which makes this identical private call from the same subclass
+                        node = cluster.nodes_manager.get_node_from_slot(
+                            slot,
+                            cluster.read_from_replicas and args[0] in READ_COMMANDS,
+                            (cluster.load_balancing_strategy if args[0] in READ_COMMANDS else None),
+                        )
+                        moved = False
+
+                    return await node.execute_command(*args, **kwargs)
+                except (BusyLoadingError, MaxConnectionsError):
+                    raise
+                except (_RedisConnectionError, _RedisTimeoutError):
+                    # Reset only the node that actually failed instead of the upstream
+                    # default (`await self.aclose()`, a full-cluster teardown that forces
+                    # every other concurrent caller through the shared reinit lock).
+                    await node.disconnect()
+                    raise
+                except (ClusterDownError, SlotNotCoveredError):
+                    await cluster.aclose()
+                    await asyncio.sleep(0.25)
+                    raise
+                except MovedError as e:
+                    cluster.reinitialize_counter += 1
+                    if cluster.reinitialize_steps and cluster.reinitialize_counter % cluster.reinitialize_steps == 0:
+                        await cluster.aclose()
+                        cluster.reinitialize_counter = 0
+                    else:
+                        cluster.nodes_manager._moved_exception = e  # pyright: ignore[reportPrivateUsage]  # mirrors upstream's own un-overridden branch; redis-py exposes no public setter for this
+                    moved = True
+                except AskError as e:
+                    redirect_addr = get_node_name(host=e.host, port=e.port)
+                    asking = True
+                except TryAgainError:
+                    if ttl < cluster.RedisClusterRequestTTL / 2:
+                        await asyncio.sleep(0.05)
+
+            raise ClusterError("TTL exhausted.")
+
+    return LiteLLMAsyncRedisCluster

litellm/constants.py

@@ -783,6 +783,7 @@ openai_compatible_endpoints: Final[list] = [
     "https://pinstripes.io/v1",
     "https://api.meta.ai/v1",
     "https://api.cognition.ai/v1",
+    "https://api.scx.ai/v1",
 ]
 
 
@@ -851,6 +852,7 @@ openai_compatible_providers: Final[list] = [
     "darkbloom",
     "meta",  # Meta Model API (Muse Spark) - JSON-configured provider
     "cognition",
+    "scx-ai",
 ]
 openai_text_completion_compatible_providers: Final[list] = [  # providers that support `/v1/completions`
     "together_ai",
@@ -1354,6 +1356,8 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
 LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
+AUTO_ROUTED_REQUEST_METADATA_KEY: Final = "_auto_routed_request"
+ROUTER_MODEL_NAME_RESPONSE_FIELD: Final = "router_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
@@ -1534,6 +1538,7 @@ TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
+SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))

litellm/integrations/custom_logger.py

@@ -60,6 +60,25 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+
+    enforces_request_content: bool = False
+    """
+    Whether this hook's ``async_pre_call_hook`` judges the request payload itself.
+
+    False for the accounting hooks, which count a request rather than read it: rate limits,
+    parallel slots, budgets, cache lookups. Those must run once per request and never once per
+    record of a batch upload, which would charge a caller once for every line of their file.
+
+    Set it to True on a hook that inspects or rejects content, so that scanning a payload which
+    is not itself a request, such as one record of a batch input file, still reaches it. A
+    ``CustomGuardrail`` does not need it; guardrails are dispatched by their own branch.
+
+    Judging content is necessary but not sufficient. A hook that also rewrites the payload for
+    routing, as the managed-files and managed-vector-store hooks do, stays False: a per-record
+    rewrite would read as a redaction and ship embedded in the record. Only the leaf class is
+    consulted, so a subclass that does not override ``async_pre_call_hook`` inherits nothing.
+    """
+
     def __init__(
         self,
         turn_off_message_logging: bool = False,

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -811,6 +811,24 @@ def _map_openai_like_exception(
             )
 
 
+_BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN: Final = re.compile(r"prompt tokens \((\d+)\) exceed model maximum \((\d+)\)")
+
+
+def _get_bedrock_mantle_context_window_message(error_str: str) -> str | None:
+    """
+    Mantle reports context overflow as a structured validation error rather than
+    the plain-text patterns Bedrock itself uses, so it needs its own detection and a
+    message clients recognize as context overflow (litellm/litellm#36546).
+    """
+    if "invalid_request_error" not in error_str and "validation_error" not in error_str:
+        return None
+    match = _BEDROCK_MANTLE_CONTEXT_WINDOW_PATTERN.search(error_str)
+    if match is None:
+        return None
+    prompt_tokens, max_tokens = match.groups()
+    return f"prompt is too long: {prompt_tokens} tokens > {max_tokens} maximum"
+
+
 def _map_bedrock_exception(
     *,
     model: str,
@@ -821,6 +839,14 @@ def _map_bedrock_exception(
     exception_provider: str,
     extra_information: str,
 ) -> None:
+    if custom_llm_provider == "bedrock_mantle":
+        mantle_context_window_message = _get_bedrock_mantle_context_window_message(error_str)
+        if mantle_context_window_message is not None:
+            raise ContextWindowExceededError(
+                message=mantle_context_window_message,
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
     if (
         "too many tokens" in error_str
         or "expected maxLength:" in error_str
@@ -2315,7 +2341,7 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
-            elif custom_llm_provider == "bedrock":
+            elif custom_llm_provider in ("bedrock", "bedrock_mantle"):
                 _map_bedrock_exception(
                     model=model,
                     original_exception=mappable_exception,

litellm/litellm_core_utils/litellm_logging.py

@@ -5615,6 +5615,37 @@ def _extract_response_obj_and_hidden_params(
     return response_obj, hidden_params
 
 
+def _autorouter_savings_for_payload(
+    request_metadata: Mapping[str, object],
+    model: str | None,
+    custom_llm_provider: str | None,
+    model_id: str | None,
+    usage_object: Mapping[str, object] | None,
+    cost_breakdown: Mapping[str, object] | None,
+) -> float | None:
+    """The auto-router savings figure for the payload, or ``None`` when there is none.
+
+    Lazy proxy import: the savings module lives with the spend trackers that own the
+    math, and SDK-only installs have no proxy package to import.
+    """
+    try:
+        from litellm.proxy.spend_tracking.savings import autorouter_savings_for_logging_payload
+    except Exception:  # noqa: BLE001  # SDK-only install: no savings driver to run
+        return None
+    try:
+        return autorouter_savings_for_logging_payload(
+            request_metadata=request_metadata,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            model_id=model_id,
+            usage_object=usage_object,
+            cost_breakdown=cost_breakdown,
+        )
+    except Exception as e:  # noqa: BLE001  # a savings figure must never fail request logging
+        verbose_logger.debug("autorouter savings skipped on logging payload: %s", e)
+        return None
+
+
 def get_standard_logging_object_payload(
     kwargs: dict | None,
     init_response_obj: Any | BaseModel | dict,
@@ -5772,6 +5803,16 @@ def get_standard_logging_object_payload(
         ):
             model_name = response_model_name
 
+        request_cost_breakdown: Final = cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost)
+        autorouter_savings: Final = _autorouter_savings_for_payload(
+            request_metadata=metadata,
+            model=model_name,
+            custom_llm_provider=custom_llm_provider,
+            model_id=_model_id,
+            usage_object=usage_dict,
+            cost_breakdown=request_cost_breakdown,
+        )
+
         payload: Final[StandardLoggingPayload] = StandardLoggingPayload(
             id=str(id),
             litellm_call_id=kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),
@@ -5802,7 +5843,8 @@ def get_standard_logging_object_payload(
             metadata=clean_metadata,
             cache_key=clean_hidden_params["cache_key"],
             response_cost=response_cost,
-            cost_breakdown=cost_breakdown_with_guardrail(logging_obj.cost_breakdown, guardrail_cost),
+            cost_breakdown=request_cost_breakdown,
+            autorouter_savings=autorouter_savings,
             total_tokens=usage_dict.get("total_tokens", 0),
             prompt_tokens=usage_dict.get("prompt_tokens", 0),
             completion_tokens=usage_dict.get("completion_tokens", 0),
@@ -5998,6 +6040,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         call_type="completion",
         stream=False,
         response_cost=response_cost,
+        autorouter_savings=None,
         response_cost_failure_debug_info=None,
         status="success",
         total_tokens=int(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT + DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT),

litellm/litellm_core_utils/ptu_pricing.py

@@ -8,7 +8,7 @@ router prices at zero serves its traffic for free.
 
 from collections.abc import Mapping
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import date, datetime, time, timezone
 from types import MappingProxyType
 from typing import Final
 
@@ -68,9 +68,17 @@ def _to_utc(parsed: datetime) -> datetime:
 
 
 def _as_utc(value: object) -> datetime | None:
-    """A model_info datetime as UTC, parsing an ISO string, else None."""
+    """A model_info datetime as UTC, parsing an ISO string, else None.
+
+    An unquoted ``2027-01-01`` in config.yaml is loaded as a ``date``, not a string, and a
+    reservation bound that fails to parse takes the whole deployment out of PTU handling,
+    so the day is read as its opening midnight rather than discarded. ``datetime`` derives
+    from ``date``, so it has to be matched first.
+    """
     if isinstance(value, datetime):
         return _to_utc(value)
+    if isinstance(value, date):
+        return datetime.combine(value, time.min, tzinfo=timezone.utc)
     if not isinstance(value, str):
         return None
     try:
@@ -84,6 +92,38 @@ def _named(reason: str, model_name: str | None) -> str:
     return reason if model_name is None else f"PTU configuration on model '{model_name}' is invalid: {reason}"
 
 
+def ptu_identity_error(
+    *, declared_id: str | None, taken: bool, current_id: str | None = None, model_name: str | None = None
+) -> str | None:
+    """Why this config-declared reservation cannot be identified, else None.
+
+    A deployment declared in config.yaml is otherwise keyed by a hash of its resolved
+    ``litellm_params``, so rotating a credential or editing an endpoint mints a second
+    identity and the reservation is charged again under it. The flat cost is keyed by that
+    id, and a charge already written is never retracted, so the duplicate is permanent.
+
+    ``current_id`` is what the deployment is keyed by today. Naming it is the difference
+    between an operator carrying their history forward and an operator inventing a fresh
+    id, which starts a second identity beside the charges already written.
+    """
+    if not declared_id:
+        return _named(
+            "model_info.id is required when PTU fields are set. Without one the deployment is "
+            "identified by a hash of its litellm_params, so rotating a credential bills the "
+            "reservation a second time under the new identity. Set it to the id this deployment "
+            f"already uses, {current_id or 'shown by GET /model/info'}, so the flat cost already "
+            "written stays under one identity; any other value starts a second one",
+            model_name,
+        )
+    if taken:
+        return _named(
+            f"model_info.id '{declared_id}' is declared on more than one deployment. Each would key "
+            "the same flat-cost row, so one reservation would go unbilled",
+            model_name,
+        )
+    return None
+
+
 def ptu_config_error(model_info: Mapping[str, object], *, model_name: str | None = None) -> str | None:
     """Why this PTU configuration cannot be honoured, else None.
 

litellm/llms/anthropic/chat/transformation.py

@@ -1827,6 +1827,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             custom_llm_provider=self.custom_llm_provider,
         )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=optional_params,
+            custom_llm_provider=self._resolved_provider,
+        )
+
         headers = self.update_headers_with_optional_anthropic_beta(headers=headers, optional_params=optional_params)
 
         # === Tool-name sanitization (single chokepoint) ===

litellm/llms/anthropic/common_utils.py

@@ -32,6 +32,12 @@ from litellm.types.llms.anthropic import (
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.proxy.model_listing import ModelInfoResponse
 
+DROP_DISABLED_THINKING_WARNING: Final = (
+    "Dropping `thinking={'type': 'disabled'}` for model=%s: thinking is always on for this model and cannot be "
+    "disabled (the alternative is a provider 400). The model will still think adaptively, its response can contain "
+    "thinking blocks, and those thinking tokens are billed as output tokens."
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
@@ -425,6 +431,35 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         """
         return AnthropicModelInfo._supports_model_capability(model, "supports_adaptive_thinking", custom_llm_provider)
 
+    @staticmethod
+    def _is_always_on_thinking_model(model: str, custom_llm_provider: str) -> bool:
+        """Whether ``model`` always thinks and rejects ``thinking.type=disabled``
+        (Fable 5 / Mythos 5 generation). The model cost map is authoritative: an
+        explicit ``thinking_always_on`` entry resolved under ``custom_llm_provider``,
+        or a ``fallback_generalizations`` rule for unmapped ids of those families.
+        """
+        return AnthropicModelInfo._supports_model_capability(model, "thinking_always_on", custom_llm_provider)
+
+    @staticmethod
+    def maybe_drop_disabled_thinking(
+        model: str,
+        optional_params: dict,  # mutable-ok: in-place out-param, same contract as AnthropicConfig._maybe_drop_speed_param
+        custom_llm_provider: str,
+    ) -> None:
+        """Omit ``thinking={'type': 'disabled'}`` for always-on-thinking models
+        (Fable 5 / Mythos 5), which 400 on it; omission is the API-documented
+        remedy and yields the model's default adaptive thinking."""
+        thinking: Final = optional_params.get("thinking")
+        if not isinstance(thinking, dict) or thinking.get("type") != "disabled":
+            return
+        if not AnthropicModelInfo._is_always_on_thinking_model(model, custom_llm_provider):
+            return
+        litellm.verbose_logger.warning(
+            DROP_DISABLED_THINKING_WARNING,
+            model,
+        )
+        optional_params.pop("thinking", None)
+
     def is_effort_used(
         self,
         optional_params: dict | None,

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -21,6 +21,7 @@ from litellm.llms.anthropic.experimental_pass_through.context_management import
 )
 from litellm.llms.anthropic.experimental_pass_through.utils import (
     is_reasoning_auto_summary_enabled,
+    local_model_name,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -358,9 +359,9 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         except Exception:
             pass
 
-        if isinstance(model, str) and model and not model.startswith("responses/"):
-            # Prefix model with "responses/" to route to OpenAI Responses API
-            completion_kwargs["model"] = f"responses/{model}"
+        if isinstance(model, str) and model and "responses/" not in model:
+            local_model: Final = model.removeprefix(f"{custom_llm_provider}/")
+            completion_kwargs["model"] = f"{custom_llm_provider}/responses/{local_model}"
 
         auto_summary: Final = is_reasoning_auto_summary_enabled()
 
@@ -616,7 +617,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         if stream:
             transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
                 completion_response,
-                model=model,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=True,
@@ -750,7 +751,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
         if stream:
             transformed_stream: Final = ANTHROPIC_ADAPTER.translate_completion_output_params_streaming(
                 completion_response,
-                model=model,
+                model=local_model_name(model, kwargs.get("custom_llm_provider")),
                 tool_name_mapping=tool_name_mapping,
                 polyfill_result=polyfill_result,
                 is_async=False,

litellm/llms/anthropic/experimental_pass_through/context_management/editors/compact.py

@@ -56,7 +56,7 @@ from ..result import PolyfillResult
 # so the summary's spend is attributed to the same scopes. The list mirrors the
 # fields populated by
 # ``LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata``.
-# ``user_api_key_model_max_budget`` / ``user_api_key_end_user_model_max_budget``
+# The three ``*_model_max_budget`` fields
 # are what ``_PROXY_VirtualKeyModelMaxBudgetLimiter`` reads post-call to update
 # the per-model spend caches, so without them the summary spend would never
 # count against the caller's model budget. ``user_api_key_end_user_id`` /
@@ -76,6 +76,7 @@ _PROPAGATED_METADATA_KEYS: Final = (
     "user_api_key_end_user_id",
     "user_api_end_user_max_budget",
     "user_api_key_model_max_budget",
+    "user_api_key_user_model_max_budget",
     "user_api_key_end_user_model_max_budget",
     "litellm_call_id",
     "litellm_parent_otel_span",
@@ -317,10 +318,14 @@ async def _check_summary_model_budget(
     The summary subrequest never passes back through ``user_api_key_auth``, so
     without this gate a caller whose ``model_max_budget`` for
     ``context_management_summary_model`` is exhausted could keep consuming that
-    model via compaction. Mirrors the ``model_max_budget`` /
-    ``end_user_model_max_budget`` enforcement that ``user_api_key_auth`` runs for
-    the client-requested model. Returns True outside the proxy or when no
+    model via compaction. Mirrors the per-model budget enforcement that
+    ``user_api_key_auth`` runs for the client-requested model. Returns True outside the proxy or when no
     per-model budget is configured.
+
+    All three scopes are checked because the summary's spend is charged to all
+    three: this file propagates the key, user and end-user budgets into the
+    subrequest's metadata, so enforcing only two of them would let compaction
+    increment a counter it can never be refused by.
     """
     if user_api_key_auth is None:
         return True
@@ -347,6 +352,25 @@ async def _check_summary_model_budget(
             )
             return False
 
+    user_model_max_budget: Final = getattr(user_api_key_auth, "user_model_max_budget", None)
+    user_id: Final = getattr(user_api_key_auth, "user_id", None)
+    if isinstance(user_model_max_budget, dict) and user_model_max_budget and user_id is not None:
+        try:
+            await model_max_budget_limiter.is_user_within_model_budget(
+                user_id=user_id,
+                user_model_max_budget=user_model_max_budget,
+                model=summary_model,
+            )
+        except litellm.BudgetExceededError:
+            return False
+        except Exception as e:  # noqa: BLE001  # a budget gate denies on any failure, as the key and end-user scopes do
+            verbose_logger.warning(
+                "compact_20260112: unexpected error during user model-budget check for summary_model=%s; denying: %s",
+                summary_model,
+                e,
+            )
+            return False
+
     end_user_model_max_budget: Final = getattr(user_api_key_auth, "end_user_model_max_budget", None)
     end_user_id: Final = getattr(user_api_key_auth, "end_user_id", None)
     if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None:

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -42,15 +42,46 @@ from .utils import AnthropicMessagesRequestUtils, mock_response
 _RESPONSES_API_PROVIDERS: Final = frozenset({"openai"})
 
 
-def _should_route_to_responses_api(custom_llm_provider: str | None) -> bool:
-    """Return True when the provider should use the Responses API path.
+def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool:
+    from litellm.main import responses_api_bridge_check
+
+    model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider)
+    return model_info.get("mode") == "responses"
+
+
+def _responses_mode_is_lost_by_prefix_strip(
+    requested_model: str, resolved_model: str, custom_llm_provider: str
+) -> bool:
+    """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped.
+
+    ``litellm.completion`` re-derives the Responses bridge from the stripped id alone, so a
+    deployment id such as ``perplexity/perplexity/sonar`` (mode ``responses``) is shadowed by the
+    chat entry ``perplexity/sonar`` and would otherwise be sent to chat/completions.
+    """
+    if requested_model == resolved_model:
+        return False
+    return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api(
+        resolved_model, custom_llm_provider
+    )
+
+
+def _should_route_to_responses_api(
+    custom_llm_provider: str | None,
+    requested_model: str | None = None,
+    resolved_model: str | None = None,
+) -> bool:
+    """Return True when the request should use the Responses API path.
 
     Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to
     opt out and route OpenAI/Azure requests through chat/completions instead.
     """
     if litellm.use_chat_completions_url_for_anthropic_messages:
         return False
-    return custom_llm_provider in _RESPONSES_API_PROVIDERS
+    if custom_llm_provider in _RESPONSES_API_PROVIDERS:
+        return True
+    if custom_llm_provider is None or requested_model is None or resolved_model is None:
+        return False
+    return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider)
 
 
 def _deployment_passes_through_anthropic_messages(model_info: object) -> bool:
@@ -533,7 +564,7 @@ def anthropic_messages_handler(
         _shared_kwargs: Final = dict(
             max_tokens=max_tokens,
             messages=messages,
-            model=model,
+            model=original_model,
             metadata=metadata,
             stop_sequences=stop_sequences,
             stream=stream,
@@ -551,7 +582,7 @@ def anthropic_messages_handler(
             custom_llm_provider=custom_llm_provider,
             **kwargs,
         )
-        if _should_route_to_responses_api(custom_llm_provider):
+        if _should_route_to_responses_api(custom_llm_provider, original_model, model):
             return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(**_shared_kwargs)
 
         # The in-gateway context_management polyfill runs inside

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -568,6 +568,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             custom_llm_provider=self._resolved_provider,
         )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=anthropic_messages_optional_request_params,
+            custom_llm_provider=self._resolved_provider,
+        )
+
         self._translate_legacy_thinking_for_adaptive_model(
             model=model,
             optional_params=anthropic_messages_optional_request_params,

litellm/llms/anthropic/experimental_pass_through/messages/utils.py

@@ -44,6 +44,7 @@ class AnthropicMessagesRequestUtils:
         filtered_params: Final = {k: v for k, v in params.items() if k in valid_keys and v is not None}
         if model is not None:
             from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+            from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 
             AnthropicConfig._maybe_drop_speed_param(
                 model=model,
@@ -51,6 +52,16 @@ class AnthropicMessagesRequestUtils:
                 drop_params=drop_params,
                 custom_llm_provider=custom_llm_provider,
             )
+            for param in ("temperature", "top_p", "top_k"):
+                if param in filtered_params:
+                    AnthropicModelInfo._apply_sampling_param(  # pyright: ignore[reportPrivateUsage]  # same gating the /chat/completions path applies; forking it would drift
+                        optional_params=filtered_params,
+                        model=model,
+                        param=param,
+                        value=filtered_params.pop(param),
+                        drop_params=drop_params,
+                        output_key=param,
+                    )
         return cast(AnthropicMessagesRequestOptionalParams, filtered_params)
 
 

litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py

@@ -19,6 +19,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
 
+from ..utils import local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
 from .transformation import LiteLLMAnthropicToResponsesAPIAdapter
 
@@ -179,7 +180,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         result: Final = await litellm.aresponses(**responses_kwargs)
 
         if stream:
-            wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
+            wrapper: Final = AnthropicResponsesStreamWrapper(
+                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+            )
             return wrapper.async_anthropic_sse_wrapper()
 
         if not isinstance(result, ResponsesAPIResponse):
@@ -257,7 +260,9 @@ class LiteLLMMessagesToResponsesAPIHandler:
         result: Final = litellm.responses(**responses_kwargs)
 
         if stream:
-            wrapper: Final = AnthropicResponsesStreamWrapper(responses_stream=result, model=model)
+            wrapper: Final = AnthropicResponsesStreamWrapper(
+                responses_stream=result, model=local_model_name(model, kwargs.get("custom_llm_provider"))
+            )
             return wrapper.async_anthropic_sse_wrapper()
 
         if not isinstance(result, ResponsesAPIResponse):

litellm/llms/anthropic/experimental_pass_through/utils.py

@@ -13,6 +13,11 @@ def prompt_cache_key_from_user_id(user_id: object) -> str | None:
     return str(user_id)[:OPENAI_MAX_PROMPT_CACHE_KEY_LENGTH] or None
 
 
+def local_model_name(model: str, custom_llm_provider: object) -> str:
+    """The id the provider itself knows, for reporting back to the caller in ``message_start``."""
+    return model.removeprefix(f"{custom_llm_provider}/") if isinstance(custom_llm_provider, str) else model
+
+
 def is_reasoning_auto_summary_enabled() -> bool:
     """Check whether the default 'summary: detailed' injection is enabled (opt-in)."""
     return litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"

litellm/llms/bedrock/chat/converse_transformation.py

@@ -39,6 +39,7 @@ from litellm.llms.anthropic.chat.transformation import (
     REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT,
     AnthropicConfig,
 )
+from litellm.llms.anthropic.common_utils import AnthropicModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.request_metadata import (
     bedrock_request_metadata_headers,
@@ -1571,6 +1572,12 @@ class AmazonConverseConfig(BaseConfig):
                     "has no thinking_blocks. The model won't use extended thinking for this turn."
                 )
 
+        AnthropicModelInfo.maybe_drop_disabled_thinking(
+            model=model,
+            optional_params=optional_params,
+            custom_llm_provider="bedrock",
+        )
+
         # Prepare and separate parameters
         (
             inference_params,

litellm/llms/openai_like/providers.json

@@ -188,5 +188,17 @@
       "max_completion_tokens": "max_tokens"
     },
     "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
+  },
+  "scx-ai": {
+    "base_url": "https://api.scx.ai/v1",
+    "api_key_env": "SCX_API_KEY",
+    "api_base_env": "SCX_API_BASE",
+    "param_mappings": {
+      "max_completion_tokens": "max_tokens"
+    },
+    "constraints": {
+      "temperature_max": 1.99
+    },
+    "supported_endpoints": ["/v1/chat/completions"]
   }
 }

litellm/llms/vertex_ai/common_utils.py

@@ -1164,21 +1164,31 @@ class VertexAITokenCounter(BaseTokenCounter):
                     original_response=result,
                 )
         else:
-            # Use standard Vertex AI (Gemini) token counter
             from litellm.llms.vertex_ai.count_tokens.handler import VertexAITokenCounter
+            from litellm.llms.vertex_ai.gemini.transformation import (
+                _gemini_convert_messages_with_history,  # pyright: ignore[reportPrivateUsage]  # shared helper already used by gemini/chat, context_caching, and vertex_and_google_ai_studio_gemini
+            )
+
+            resolved_contents: Final = (
+                contents
+                if contents is not None
+                else _gemini_convert_messages_with_history(
+                    messages=messages or []  # mutable-ok: fallback for None messages; helper signature requires list
+                )
+            )
 
             count_tokens_params: Final = {
                 "model": model_to_use,
-                "contents": contents,
+                "contents": resolved_contents,
             }
             count_tokens_params_request.update(count_tokens_params)
             result = await VertexAITokenCounter().acount_tokens(
                 **count_tokens_params_request,
             )
 
-            if result is not None:
+            if result is not None and "totalTokens" in result:
                 return TokenCountResponse(
-                    total_tokens=result.get("totalTokens", 0),
+                    total_tokens=result["totalTokens"],
                     request_model=request_model,
                     model_used=model_to_use,
                     tokenizer_type=result.get("tokenizer_used", ""),

litellm/model_prices_and_context_window_backup.json

@@ -1232,6 +1232,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
+        "thinking_always_on": true,
         "supports_function_calling": true,
         "supports_vision": true,
         "supports_prompt_caching": false,
@@ -1404,6 +1405,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1440,6 +1442,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1476,6 +1479,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1512,6 +1516,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -3021,6 +3026,7 @@
         "cache_creation_input_token_cost_above_1hr": 2e-05,
         "cache_read_input_token_cost": 1e-06,
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -4875,6 +4881,38 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "azure/gpt-audio-mini": {
+        "deprecation_date": "2027-04-06",
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "azure",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
     "azure/gpt-audio-mini-2025-10-06": {
         "deprecation_date": "2027-04-06",
         "input_cost_per_audio_token": 1e-05,
@@ -5088,6 +5126,38 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "azure/gpt-realtime-mini": {
+        "cache_creation_input_audio_token_cost": 3e-07,
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_image": 8e-07,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "azure",
+        "max_input_tokens": 32000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "realtime",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "azure/gpt-realtime-mini-2025-10-06": {
         "cache_creation_input_audio_token_cost": 3e-07,
         "cache_read_input_token_cost": 6e-08,
@@ -12780,6 +12850,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -19491,106 +19562,6 @@
         },
         "web_search_billing_unit": "per_query"
     },
-    "gemini-3.1-flash-lite-image": {
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "litellm_provider": "vertex_ai-language-models",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": false,
-        "supports_prompt_caching": true,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
-    "gemini/gemini-3.1-flash-lite-image": {
-        "rpm": 1000,
-        "tpm": 4000000,
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "input_cost_per_token_batches": 1.25e-07,
-        "litellm_provider": "gemini",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "output_cost_per_token_batches": 7.5e-07,
-        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": true,
-        "supports_prompt_caching": false,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
-    "vertex_ai/gemini-3.1-flash-lite-image": {
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "litellm_provider": "vertex_ai-language-models",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": false,
-        "supports_prompt_caching": true,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
     "gemini-3.1-flash-image": {
         "deprecation_date": "2027-05-28",
         "input_cost_per_image": 0.00056,
@@ -19668,6 +19639,44 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.1-flash-lite-image": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "gemini-3.1-flash-lite-preview": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_audio_token": 5e-07,
@@ -21498,6 +21507,42 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.1-flash-lite-image": {
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "rpm": 1000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "tpm": 4000000
+    },
     "gemini/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -26034,33 +26079,33 @@
         "supports_minimal_reasoning_effort": true
     },
     "gpt-5.6": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
-        "cache_creation_input_token_cost_flex": 3.125e-06,
-        "cache_creation_input_token_cost_priority": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
-        "cache_read_input_token_cost_flex": 2.5e-07,
-        "cache_read_input_token_cost_priority": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
-        "input_cost_per_token_above_272k_tokens_flex": 5e-06,
-        "input_cost_per_token_batches": 2.5e-06,
-        "input_cost_per_token_flex": 2.5e-06,
-        "input_cost_per_token_priority": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
+        "cache_creation_input_token_cost_flex": 2.5e-06,
+        "cache_creation_input_token_cost_priority": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
+        "cache_read_input_token_cost_flex": 2e-07,
+        "cache_read_input_token_cost_priority": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
+        "input_cost_per_token_above_272k_tokens_flex": 4e-06,
+        "input_cost_per_token_batches": 2e-06,
+        "input_cost_per_token_flex": 2e-06,
+        "input_cost_per_token_priority": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
-        "output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
-        "output_cost_per_token_batches": 1.5e-05,
-        "output_cost_per_token_flex": 1.5e-05,
-        "output_cost_per_token_priority": 6e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
+        "output_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_flex": 1e-05,
+        "output_cost_per_token_priority": 4e-05,
         "regional_processing_uplift_multiplier_eu": 1.1,
         "regional_processing_uplift_multiplier_us": 1.1,
         "search_context_cost_per_query": {
@@ -26097,33 +26142,33 @@
         "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-sol": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
-        "cache_creation_input_token_cost_flex": 3.125e-06,
-        "cache_creation_input_token_cost_priority": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
-        "cache_read_input_token_cost_flex": 2.5e-07,
-        "cache_read_input_token_cost_priority": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
-        "input_cost_per_token_above_272k_tokens_flex": 5e-06,
-        "input_cost_per_token_batches": 2.5e-06,
-        "input_cost_per_token_flex": 2.5e-06,
-        "input_cost_per_token_priority": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
+        "cache_creation_input_token_cost_flex": 2.5e-06,
+        "cache_creation_input_token_cost_priority": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
+        "cache_read_input_token_cost_flex": 2e-07,
+        "cache_read_input_token_cost_priority": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
+        "input_cost_per_token_above_272k_tokens_flex": 4e-06,
+        "input_cost_per_token_batches": 2e-06,
+        "input_cost_per_token_flex": 2e-06,
+        "input_cost_per_token_priority": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
-        "output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
-        "output_cost_per_token_batches": 1.5e-05,
-        "output_cost_per_token_flex": 1.5e-05,
-        "output_cost_per_token_priority": 6e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
+        "output_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_flex": 1e-05,
+        "output_cost_per_token_priority": 4e-05,
         "regional_processing_uplift_multiplier_eu": 1.1,
         "regional_processing_uplift_multiplier_us": 1.1,
         "search_context_cost_per_query": {
@@ -26365,19 +26410,19 @@
         "supports_parallel_function_calling": true
     },
     "daybreak-blue-latest": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/responses"
@@ -31140,6 +31185,23 @@
         "supports_video_input": true,
         "supports_vision": true
     },
+    "moonshot/kimi-k3": {
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "moonshot",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://platform.kimi.ai/docs/pricing/chat-k3",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "moonshot/kimi-latest": {
         "cache_read_input_token_cost": 1.5e-07,
         "deprecation_date": "2026-01-28",
@@ -36724,6 +36786,40 @@
         "supports_vision": true,
         "source": "https://cloud.sambanova.ai/plans/pricing"
     },
+    "scx-ai/GLM-5.2": {
+        "cache_read_input_token_cost": 2.2e-07,
+        "input_cost_per_token": 6.1e-07,
+        "litellm_provider": "scx-ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.98e-06,
+        "source": "https://scx.ai/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "scx-ai/Qwen3.8-Max": {
+        "cache_read_input_token_cost": 2.1e-07,
+        "input_cost_per_token": 1.65e-06,
+        "litellm_provider": "scx-ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 4.99e-06,
+        "source": "https://scx.ai/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "snowflake/claude-3-5-sonnet": {
         "litellm_provider": "snowflake",
         "max_input_tokens": 200000,
@@ -40365,6 +40461,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -40398,6 +40495,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -41006,6 +41104,44 @@
         "supports_reasoning": false,
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models"
     },
+    "vertex_ai/gemini-3.1-flash-lite-image": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "vertex_ai/gemini-3.1-flash-lite-preview": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_audio_token": 5e-07,
@@ -48545,6 +48681,156 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "us.openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5.5e-06,
+        "input_cost_per_token_above_272k_tokens": 1.1e-05,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
+        "cache_read_input_token_cost": 5.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
+        "output_cost_per_token": 3.3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "us.openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2.2e-06,
+        "input_cost_per_token_above_272k_tokens": 4.4e-06,
+        "cache_creation_input_token_cost": 2.75e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
+        "cache_read_input_token_cost": 2.2e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
+        "output_cost_per_token": 1.32e-05,
+        "output_cost_per_token_above_272k_tokens": 1.98e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_272k_tokens": 4e-06,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 5e-06,
+        "cache_read_input_token_cost": 2e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 4e-07,
+        "output_cost_per_token": 1.2e-05,
+        "output_cost_per_token_above_272k_tokens": 1.8e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "us.openai.gpt-5.6-luna": {
+        "input_cost_per_token": 2.2e-07,
+        "input_cost_per_token_above_272k_tokens": 4.4e-07,
+        "cache_creation_input_token_cost": 2.75e-07,
+        "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
+        "cache_read_input_token_cost": 2.2e-08,
+        "cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
+        "output_cost_per_token": 1.32e-06,
+        "output_cost_per_token_above_272k_tokens": 1.98e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-luna": {
+        "input_cost_per_token": 2e-07,
+        "input_cost_per_token_above_272k_tokens": 4e-07,
+        "cache_creation_input_token_cost": 2.5e-07,
+        "cache_creation_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost": 2e-08,
+        "cache_read_input_token_cost_above_272k_tokens": 4e-08,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_272k_tokens": 1.8e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "bedrock_mantle/openai.gpt-5.5": {
         "input_cost_per_token": 5.5e-06,
         "cache_read_input_token_cost": 5.5e-07,
@@ -49754,6 +50040,7 @@
         },
         "source": "https://docs.claude.com/en/docs/about-claude/models/overview",
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -49789,6 +50076,7 @@
         },
         "source": "https://docs.claude.com/en/docs/about-claude/models/overview",
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -49967,6 +50255,14 @@
                     "supports_adaptive_thinking": true
                 }
             },
+            {
+                "name": "claude-always-on-thinking",
+                "pattern": "claude-(?:fable|mythos)-",
+                "description": "Any Claude Fable or Mythos id, under any provider namespace and any version. These families always think and reject thinking.type=disabled with a 400; the Anthropic transformations omit the param instead, so the model falls back to its default adaptive thinking.",
+                "model_info": {
+                    "thinking_always_on": true
+                }
+            },
             {
                 "name": "claude-mid-conversation-system",
                 "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",

litellm/provider_endpoints_support_backup.json

@@ -2027,6 +2027,23 @@
         "interactions": true
       }
     },
+    "scx-ai": {
+      "display_name": "SCX.ai (`scx-ai`)",
+      "url": "https://docs.litellm.ai/docs/providers/scx_ai",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "snowflake": {
       "display_name": "Snowflake (`snowflake`)",
       "url": "https://docs.litellm.ai/docs/providers/snowflake",

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -39,6 +39,7 @@ from litellm.proxy._types import (
     SpecialMCPServerName,
     SpecialMCPServerNames,
     UserAPIKeyAuth,
+    user_api_key_has_admin_view,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.auth.user_api_key_auth import (
@@ -160,7 +161,7 @@ def _is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> b
     """True when this auth is a keyless subject admitted by the gateway session / bridge user
     path, as opposed to a JWT or other keyless auth that merely lacks a ``team_id``.
 
-    Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``_reload_admitted_user``. It
+    Reads the server-only ``mcp_admitted_user_subject`` field, set only by ``reload_admitted_user``. It
     is deliberately NOT a ``metadata`` key, which is caller-controlled at key creation and so forgeable
     on a personal key to gain the team grant union or dodge the egress scrub; this field cannot be."""
     return user_api_key_auth is not None and user_api_key_auth.mcp_admitted_user_subject is True
@@ -812,7 +813,7 @@ class MCPRequestHandler:
 
         Identity-only sibling of :meth:`_admit_dcr_bridge_delegate`: the session token seals no
         upstream credential (those are vaulted per user, resolved at egress), so authorization is
-        resolved fresh via :meth:`_reload_admitted_user` + the centralized policy gate rather than a
+        resolved fresh via :meth:`reload_admitted_user` + the centralized policy gate rather than a
         mint-time snapshot. Pre-DB gates (size, IP, route allowlist) run first, mirroring the standard
         pipeline. Fails closed with the requested scope's ``invalid_token`` challenge on an expired,
         tampered, foreign, or refresh token, or a missing/deactivated/policy-rejected user."""
@@ -835,7 +836,7 @@ class MCPRequestHandler:
         match result:
             case SessionBearerAdmitted():
                 try:
-                    admitted: Final = await MCPRequestHandler._reload_admitted_user(result.principal.user_id)
+                    admitted: Final = await MCPRequestHandler.reload_admitted_user(result.principal.user_id)
                     admitted.mcp_session_resource_server_id = result.principal.resource_server_id
                     await MCPRequestHandler._enforce_admitted_live_policy(
                         admitted=admitted, request=request, route=route
@@ -893,12 +894,12 @@ class MCPRequestHandler:
             case "key_hash":
                 return await MCPRequestHandler._reload_admitted_key(identity.subject)
             case "user_id":
-                return await MCPRequestHandler._reload_admitted_user(identity.subject)
+                return await MCPRequestHandler.reload_admitted_user(identity.subject)
             case _:
                 assert_never(identity.subject_type)
 
     @staticmethod
-    async def _reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
+    async def reload_admitted_user(user_id: str) -> UserAPIKeyAuth:
         """Reload the live user an interactively-minted envelope references and admit them as themselves.
 
         The user's own object permission and ``org_id`` ride on the returned ``UserAPIKeyAuth``, and the
@@ -1785,11 +1786,14 @@ class MCPRequestHandler:
             global_mcp_server_manager,
         )
 
-        # An OPEN channel (allow_all_keys, the user's own BYOM) makes the server REACHABLE through the
-        # user, though no grant source names it — without this the union returns [], listable but
-        # uninvokable. Reachability is ALL it confers, NOT a ceiling waiver: the user's own
-        # mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do on an allow_all server.
-        reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(auth)
+        # An OPEN channel (allow_all_keys, the user's own BYOM, an unscoped admin-view role) makes the
+        # server REACHABLE through the user, though no grant source names it — without this the union
+        # returns [], listable but uninvokable. Reachability is ALL it confers, NOT a ceiling waiver:
+        # the user's own mcp_tool_permissions and org tool ceiling still bind, exactly as a key's do
+        # on an allow_all server or an admin key's do on any server.
+        reachable_via_open_channel: Final = server_id in await global_mcp_server_manager.operator_open_server_ids(
+            auth
+        ) or await MCPRequestHandler.admin_view_unscoped(auth)
 
         allowed: Final[set[str]] = set()
         for source, granted in await MCPRequestHandler.admitted_source_grants(auth):
@@ -2723,6 +2727,32 @@ class MCPRequestHandler:
         entitled_servers: Final = await MCPRequestHandler._get_allowed_mcp_servers_for_user(user_api_key_auth)
         return entitled_servers is None or len(entitled_servers) > 0
 
+    @staticmethod
+    async def admin_view_unscoped(user_api_key_auth: UserAPIKeyAuth | None = None) -> bool:
+        """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
+        credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
+
+        Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
+        ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
+        admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
+        [] by DB default, so for that shape the row binds through the entitlement ceiling instead
+        (any non-empty entitlement, or an unresolved one, disqualifies), exactly as
+        ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
+        server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
+        channel in ``_resolve_admitted_subject_tools`` both consult it, so the two axes cannot
+        disagree."""
+        if user_api_key_auth is None or not user_api_key_has_admin_view(user_api_key_auth):
+            return False
+        object_permission: Final = user_api_key_auth.object_permission
+        credential_scoped: Final = (
+            not _is_mcp_admitted_user_subject(user_api_key_auth)
+            and object_permission is not None
+            and object_permission.mcp_servers is not None
+        )
+        if credential_scoped:
+            return False
+        return not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
+
     @staticmethod
     async def _apply_user_tool_ceiling(
         allowed_tools: Sequence[str] | None,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -750,6 +750,55 @@ def _redirect_to_upstream_authorize(
     return RedirectResponse(urlunparse(parsed_auth_url._replace(query=urlencode(merged_params))))
 
 
+def _bridge_access_denied_redirect(redirect_uri: str, state: str, mcp_server: MCPServer) -> RedirectResponse:
+    """RFC 6749 section 4.1.2.1 denial for the interactive bridge authorize, delivered to the
+    already-validated client redirect_uri so a DCR client surfaces the failure at connect time."""
+    server_label: Final = mcp_server.alias or mcp_server.server_name or mcp_server.server_id
+    params: Final = {
+        "error": "access_denied",
+        "error_description": (
+            f"the signed-in user has no access to MCP server '{server_label}' on this gateway; "
+            "grant it through a team or user object permission, or mark the server allow_all_keys"
+        ),
+        **({"state": state} if state else {}),
+    }
+    return RedirectResponse(_append_query_params(redirect_uri, params), status_code=302)
+
+
+async def _bridge_authorize_access_denial(
+    litellm_user_id: str,
+    mcp_server: MCPServer,
+    redirect_uri: str,
+    state: str,
+) -> RedirectResponse | None:
+    """The denial redirect for a signed-in user who cannot reach the target server, or None to proceed.
+
+    Admits the user exactly as MCP egress will (the same ``reload_admitted_user`` constructor and the
+    same ``get_allowed_mcp_servers`` resolver), so an envelope is minted only when the resulting
+    session can actually list and call the server's tools. Without this gate the flow completes, the
+    client shows connected, and every tool request fail-closes to an empty list with nothing telling
+    the operator why. An availability fault (5xx, e.g. a DB outage's 503) propagates; an unknown or
+    deactivated user denies like a missing grant, fail closed.
+    """
+    from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
+        MCPRequestHandler,
+    )
+    from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+        global_mcp_server_manager,
+    )
+
+    try:
+        admitted: Final = await MCPRequestHandler.reload_admitted_user(litellm_user_id)
+    except HTTPException as exc:
+        if exc.status_code >= 500:
+            raise
+        return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
+    allowed_server_ids: Final = await global_mcp_server_manager.get_allowed_mcp_servers(admitted)
+    if mcp_server.server_id in allowed_server_ids:
+        return None
+    return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
+
+
 async def authorize_with_server(
     request: Request,
     mcp_server: MCPServer,
@@ -819,6 +868,14 @@ async def authorize_with_server(
         litellm_user_id = _user_id_from_session_cookie(request)
         if litellm_user_id is None:
             return _redirect_to_litellm_login(request)
+        denial: Final = await _bridge_authorize_access_denial(
+            litellm_user_id=litellm_user_id,
+            mcp_server=mcp_server,
+            redirect_uri=redirect_uri,
+            state=state,
+        )
+        if denial is not None:
+            return denial
 
     encoded_state: Final = encode_state_with_base_url(
         base_url=base_url,

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -2943,17 +2943,14 @@ class MCPServerManager:
         2. If admin and no object_permission, return all servers
         3. Otherwise, use standard permission checks
         """
-        from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
-
         allow_all_server_ids: Final = self.get_allow_all_keys_server_ids()
 
         # A keyless admitted subject is resolved per grant source, and channel decisions that are
         # absolute for a scoped KEY credential are not absolute for it: its own opt-out silences its
-        # own source (handled per source in the resolver), never its teams' grants, and its admin
-        # role does not swallow the grant model — a session bearer is a third-party client
-        # credential, not the dashboard, so an admin signing in through the connect flow gets their
-        # grants like anyone else rather than handing the client the full registry ahead of every
-        # per-team org ceiling.
+        # own source (handled per source in the resolver), never its teams' grants. Its admin role
+        # rides the HUMAN, not the credential: an admin's session resolves the same registry their
+        # dashboard shows (connect-page parity), bounded like an admin key by explicit
+        # object_permission scope, the entitlement ceiling, and the session resource scope below.
         is_admitted_subject: Final = _is_mcp_admitted_user_subject(user_api_key_auth)
 
         # The key explicitly opted out of every MCP server. Return zero before
@@ -2982,26 +2979,16 @@ class MCPServerManager:
         )
 
         try:
-            # If admin but NO explicit object permission, get all servers (never for an admitted
-            # subject — see is_admitted_subject above)
-            if (
-                user_api_key_auth
-                and not is_admitted_subject
-                and _user_has_admin_view(user_api_key_auth)
-                and not has_explicit_object_permission
-                # An entitlement attached to the HUMAN binds them whatever their role: it is the
-                # person's scope, not the credential's, so an admin role is not a waiver of it. An
-                # UNRESOLVED entitlement also skips the shortcut, so the resolver denies rather than
-                # handing over the whole registry on a transient fault.
-                and not await MCPRequestHandler._user_places_mcp_ceiling(user_api_key_auth)
-            ):
-                verbose_logger.debug("Admin user without explicit object_permission - returning all servers")
-                return list(self.get_registry().keys())
-
-            # Get allowed servers from object permissions (respects object_permission even for admins)
-            allowed_mcp_servers: Final = await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth)
-            verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", allowed_mcp_servers)
-            combined_servers: Final = set(allowed_mcp_servers)
+            # Admin view with no explicit object permission and no entitlement ceiling resolves the
+            # whole registry, for keys AND admitted session subjects alike (one predicate owns the
+            # question). Seeded into the union rather than returned early so the session resource
+            # scope below still bounds a per-server envelope held by an admin.
+            combined_servers: Final = (
+                set(self.get_registry().keys())
+                if await MCPRequestHandler.admin_view_unscoped(user_api_key_auth)
+                else set(await MCPRequestHandler.get_allowed_mcp_servers(user_api_key_auth))
+            )
+            verbose_logger.debug("Allowed MCP Servers for user api key auth: %s", combined_servers)
             combined_servers.update(
                 await self.operator_open_server_ids(
                     user_api_key_auth,

litellm/proxy/_experimental/mcp_server/oauth_utils.py

@@ -180,6 +180,22 @@ def well_known_root_suffix() -> str:
     return "" if root == "/" else root
 
 
+def get_route_relative_request_path(scope: Scope) -> str:
+    """The request path the MCP route shapes are written against: the raw ASGI path with the
+    deployment's ``root_path`` removed.
+
+    ``scope["path"]`` and ``_original_path`` are both raw request-line paths, so on a sub-path
+    deployment they still carry the ``SERVER_ROOT_PATH`` prefix (``/litellm/{server}/mcp``) while
+    every route shape compared against them is root-relative. Mirrors the segment-boundary strip in
+    :func:`litellm.proxy.auth.auth_utils.get_request_route`, which the rest of the MCP auth path
+    already routes through, so ``/litellmfoo`` is not truncated under ``root_path=/litellm``."""
+    raw_path = str(scope.get("_original_path") or scope.get("path", "") or "")
+    root_path = str(scope.get("app_root_path") or scope.get("root_path") or "").rstrip("/")
+    if root_path and (raw_path == root_path or raw_path.startswith(f"{root_path}/")):
+        return raw_path[len(root_path) :]
+    return raw_path
+
+
 def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str:
     """The per-server protected-resource metadata URL matching the spelling the request
     arrived on, so a strict RFC 9728 client resolves the same route the proxy registered.
@@ -188,7 +204,7 @@ def get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str
     the route decorators insert it (see :func:`well_known_root_suffix`)."""
     request: Final = Request(scope)
     base_url: Final = get_request_base_url(request)
-    _path: Final = scope.get("_original_path") or scope.get("path", "") or ""
+    _path: Final = get_route_relative_request_path(scope)
 
     if _path.startswith(f"/{server_name}/mcp"):
         return f"{base_url}/.well-known/oauth-protected-resource{well_known_root_suffix()}/{server_name}/mcp"

litellm/proxy/_experimental/mcp_server/server.py

@@ -51,6 +51,8 @@ from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     _redact_mcp_resource_url,
     get_passthrough_www_authenticate,
+    get_route_relative_request_path,
+    well_known_root_suffix,
 )
 from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_DESCRIPTION,
@@ -3782,14 +3784,15 @@ if MCP_AVAILABLE:
 
                     request = StarletteRequest(scope)
                     base_url = get_request_base_url(request)
-                    _path = scope.get("_original_path") or scope.get("path", "") or ""
+                    _path = get_route_relative_request_path(scope)
 
                     # Pick the well-known AS-metadata form that matches the inbound route
                     # so strict RFC 9728 §3.2 clients can resolve it correctly.
+                    as_metadata_root = f"{base_url}/.well-known/oauth-authorization-server{well_known_root_suffix()}"
                     if _path.startswith(f"/mcp/{server_name}"):
-                        _as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}"
+                        _as_url = f"{as_metadata_root}/mcp/{server_name}"
                     else:
-                        _as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}"
+                        _as_url = f"{as_metadata_root}/{server_name}"
                     authorization_uri = f'Bearer authorization_uri="{_as_url}"'
 
                     raise HTTPException(

litellm/proxy/_experimental/mcp_server/ui_session_utils.py

@@ -91,7 +91,7 @@ async def admitted_user_context(user_api_key_auth: UserAPIKeyAuth) -> UserAPIKey
     )
 
     try:
-        admitted: Final = await MCPRequestHandler._reload_admitted_user(user_id)
+        admitted: Final = await MCPRequestHandler.reload_admitted_user(user_id)
     except HTTPException as e:
         verbose_logger.warning("MCP dashboard session: admitted-subject reload failed for %s: %s", user_id, e.detail)
         return None

litellm/proxy/_types.py

@@ -15,7 +15,7 @@ from pydantic import (
     field_validator,
     model_validator,
 )
-from typing_extensions import NotRequired, Required, TypedDict
+from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 
 from litellm._uuid import uuid
 from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS
@@ -2805,10 +2805,14 @@ class UserAPIKeyAuth(LiteLLM_VerificationTokenView):  # the expected response ob
     user_email: str | None = None
     user_spend: float | None = None
     user_max_budget: float | None = None
+    # Values stay `object` rather than BudgetConfig: this is the raw JSON column,
+    # and validating it here would make one malformed row fail auth outright.
+    # resolve_model_budget validates the single entry a request actually needs.
+    user_model_max_budget: dict[str, object] | None = None
     request_route: str | None = None
     is_session_token: bool = False
     # Server-only marker set exclusively by the MCP gateway admission path
-    # (_reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session
+    # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session
     # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key
     # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union
     # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization.
@@ -2982,6 +2986,8 @@ class UserInfoV2Response(LiteLLMPydanticObjectBase):
     sso_user_id: str | None = None
     teams: list[str] = []  # Just team IDs, not full team objects
     object_permission: LiteLLM_ObjectPermissionTable | None = None
+    model_max_budget: dict | None = None
+    model_max_budget_usage: dict | None = None
 
 
 from litellm.models.config import LiteLLM_Config as LiteLLM_Config  # noqa: E402
@@ -3531,6 +3537,7 @@ class SpendLogsMetadata(TypedDict):
     max_retries: int | None  # Max retries configured for this request
     cost_breakdown: CostBreakdown | None  # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
     compression_savings: CompressionSavingsMetadata | None
+    autorouter_savings: ReadOnly[float | None]  # stamped by the logging payload; None = not auto-routed
 
 
 class SpendLogsPayload(TypedDict):

litellm/proxy/auth/auth_utils.py

@@ -1801,7 +1801,7 @@ def _format_model_candidates(
     return candidates
 
 
-def _request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool:
+def request_dispatched_to_pass_through_endpoint(request: Request | None) -> bool:
     """Whether FastAPI resolved this request to a user-defined pass-through handler.
 
     Reads the marker set by ``create_pass_through_route`` off the dispatched endpoint
@@ -1842,7 +1842,7 @@ def get_model_from_request(
     and does not carry the marker. Built-in provider passthrough routes
     (``/vertex_ai``, ``/gemini``, ...) are separate handlers and keep model enforcement.
     """
-    if _request_dispatched_to_pass_through_endpoint(request):
+    if request_dispatched_to_pass_through_endpoint(request):
         return None
 
     candidates: Final = _extract_model_candidates_from_request(

litellm/proxy/auth/user_api_key_auth.py

@@ -11,6 +11,7 @@ import asyncio
 import fnmatch
 import re
 import secrets
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from typing import Any, Final, NamedTuple, Protocol, Union, cast
 
@@ -186,6 +187,62 @@ class _KeyModelBudgetLimiter(Protocol):
     async def get_fallback_model_within_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> str | None: ...
 
 
+class _UserModelBudgetLimiter(Protocol):
+    async def is_user_within_model_budget(
+        self, user_id: str, user_model_max_budget: Mapping[str, object], model: str
+    ) -> bool: ...
+
+
+async def _read_user_model_max_budget(
+    user_id: str | None,
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+    parent_otel_span: object,
+    proxy_logging_obj: ProxyLogging,
+) -> dict | None:
+    """The user row's `model_max_budget`, or None when the row cannot be read.
+
+    A user whose row is missing must not be refused: this is a budget lookup,
+    and the main auth path likewise treats an unreadable user as no user.
+    """
+    if user_id is None or prisma_client is None:
+        return None
+    try:
+        user_obj: Final = await get_user_object(
+            user_id=user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+            parent_otel_span=parent_otel_span,  # pyright: ignore[reportArgumentType]  # Span is a runtime union, not usable in an annotation here
+            proxy_logging_obj=proxy_logging_obj,
+        )
+    except Exception as e:  # noqa: BLE001  # mirrors the main path's tolerance
+        verbose_logger.debug("Unable to read user for the per-model budget check: %s", e)
+        return None
+    return getattr(user_obj, "model_max_budget", None)
+
+
+async def _check_user_model_budget(
+    valid_token: UserAPIKeyAuth,
+    model_max_budget_limiter: _UserModelBudgetLimiter,
+    models: list[str],
+) -> None:
+    """Enforce the internal user's own `model_max_budget` across the request's models.
+
+    Separate from the key check: a user's per-model budget caps every key they
+    own, so a caller cannot escape it by minting another key.
+    """
+    user_model_max_budget: Final = valid_token.user_model_max_budget
+    if valid_token.user_id is None or not isinstance(user_model_max_budget, Mapping) or not user_model_max_budget:
+        return
+    for model_name in models:
+        await model_max_budget_limiter.is_user_within_model_budget(
+            user_id=valid_token.user_id,
+            user_model_max_budget=user_model_max_budget,
+            model=model_name,
+        )
+
+
 async def _check_key_model_budget_with_fallback(
     valid_token: UserAPIKeyAuth,
     model_max_budget_limiter: _KeyModelBudgetLimiter,
@@ -1390,6 +1447,7 @@ async def _user_api_key_auth_builder(
                         end_user_id=end_user_id,
                         user_tpm_limit=(user_object.tpm_limit if user_object is not None else None),
                         user_rpm_limit=(user_object.rpm_limit if user_object is not None else None),
+                        user_model_max_budget=(user_object.model_max_budget if user_object is not None else None),
                         team_member_rpm_limit=(
                             team_membership.safe_get_team_member_rpm_limit() if team_membership is not None else None
                         ),
@@ -1427,6 +1485,13 @@ async def _user_api_key_auth_builder(
                         if auto_registered is not None:
                             auto_registered.jwt_claims = jwt_claims
                             auto_registered.user_email = user_email
+                            # The auto-registered token is built from the new key's
+                            # columns, which carry no user budget. Carry over the
+                            # already-loaded user row rather than re-reading it, or
+                            # the budget check below has nothing to enforce.
+                            auto_registered.user_model_max_budget = (
+                                user_object.model_max_budget if user_object is not None else None
+                            )
                             valid_token = auto_registered
                             api_key = valid_token.token or ""
 
@@ -1458,6 +1523,28 @@ async def _user_api_key_auth_builder(
                             valid_token.project_metadata = _jwt_project_obj.metadata
                             valid_token.project_alias = _jwt_project_obj.project_alias
 
+                    # JWT auth returns here rather than falling through to the
+                    # virtual-key checks below, so the user's per-model budget
+                    # has to be enforced on this path too. Without it the
+                    # post-call increment still charges the counter and nothing
+                    # ever reads it, which is worse than not tracking at all.
+                    # Guarded by the same flag the virtual-key path uses, or a
+                    # zero-cost model would be refused here and allowed there,
+                    # while the log above claims all budget checks were skipped.
+                    if not skip_budget_checks:
+                        await _check_user_model_budget(
+                            valid_token=cast(UserAPIKeyAuth, valid_token),
+                            model_max_budget_limiter=model_max_budget_limiter,
+                            models=_get_model_names_for_budget_checks(
+                                model=_get_model_from_request_context(
+                                    request_data=request_data,
+                                    route=route,
+                                    request=request,
+                                    llm_router=llm_router,
+                                )
+                            ),
+                        )
+
                     return cast(UserAPIKeyAuth, valid_token)
 
         #### ELSE ####
@@ -1811,6 +1898,12 @@ async def _user_api_key_auth_builder(
                     )
                     user_obj = None
 
+                if user_obj is not None:
+                    # The joint verification-token view carries the key's columns only, so the
+                    # user's own per-model budget reaches enforcement and the post-call
+                    # increment through the row fetched here.
+                    valid_token.user_model_max_budget = user_obj.model_max_budget
+
                 if (
                     user_obj is not None
                     and isinstance(user_obj.metadata, dict)
@@ -1974,6 +2067,14 @@ async def _user_api_key_auth_builder(
                         )
                         current_models = _get_model_names_for_budget_checks(model=current_model)
 
+                    # Check 5a. Internal user model_max_budget
+                    if current_models:
+                        await _check_user_model_budget(
+                            valid_token=valid_token,
+                            model_max_budget_limiter=model_max_budget_limiter,
+                            models=current_models,
+                        )
+
                     # Check 5b. End-user model max budget
                     end_user_mmb: Final = valid_token.end_user_model_max_budget
                     if (
@@ -2757,6 +2858,7 @@ async def _return_user_api_key_auth_obj(
             user_email=user_obj.user_email,
             user_spend=getattr(user_obj, "spend", None),
             user_max_budget=getattr(user_obj, "max_budget", None),
+            user_model_max_budget=getattr(user_obj, "model_max_budget", None),
         )
     if user_obj is not None and _is_user_proxy_admin(user_obj=user_obj):
         user_api_key_kwargs.update(
@@ -3020,10 +3122,21 @@ async def _run_post_custom_auth_checks(
     )
     current_models = _get_model_names_for_budget_checks(model=current_model)
 
+    # A zero-cost model cannot move any counter, so refusing it means refusing on
+    # spend some other model accrued. The JWT and virtual-key paths already skip
+    # every budget check for these; this path did not, so the same request could
+    # be refused under custom auth and served under the other two.
+    skip_budget_checks: Final = (
+        _is_model_cost_zero(model=current_model, llm_router=llm_router)
+        if current_model is not None and llm_router is not None
+        else False
+    )
+
     # 3. Check key-level model_max_budget
     max_budget_per_model: Final = valid_token.model_max_budget
     if (
-        max_budget_per_model is not None
+        not skip_budget_checks
+        and max_budget_per_model is not None
         and isinstance(max_budget_per_model, dict)
         and len(max_budget_per_model) > 0
         and current_models
@@ -3050,10 +3163,33 @@ async def _run_post_custom_auth_checks(
         )
         current_models = _get_model_names_for_budget_checks(model=current_model)
 
+    # 3b. Attach and check the internal user's model_max_budget.
+    # Custom auth builds its own token, so unlike the main path nothing has
+    # loaded the user row yet. The attach is unconditional because the post-call
+    # spend hook reads this field off the token: gating it on the same condition
+    # as enforcement would leave the user's counter uncharged whenever this
+    # request was not itself enforceable, which is the untracked-spend bug this
+    # PR exists to fix.
+    user_budget: Final = await _read_user_model_max_budget(
+        user_id=valid_token.user_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=parent_otel_span,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    valid_token.user_model_max_budget = user_budget  # rebind-ok: the spend hook reads it off this token
+    if not skip_budget_checks and current_models:
+        await _check_user_model_budget(
+            valid_token=valid_token,
+            model_max_budget_limiter=model_max_budget_limiter,
+            models=current_models,
+        )
+
     # 4. Check end-user model_max_budget
     end_user_mmb: Final = valid_token.end_user_model_max_budget
     if (
-        end_user_mmb is not None
+        not skip_budget_checks
+        and end_user_mmb is not None
         and isinstance(end_user_mmb, dict)
         and len(end_user_mmb) > 0
         and current_models

litellm/proxy/common_request_processing.py

@@ -22,12 +22,14 @@ import litellm
 from litellm._logging import _redact_string, verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.constants import (
+    AUTO_ROUTED_REQUEST_METADATA_KEY,
     DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE,
     DEFAULT_MAX_RECURSE_DEPTH,
     LITELLM_DETAILED_TIMING,
     LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED,
     MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
     RETURN_RAW_MODEL_NAME_METADATA_KEY,
+    ROUTER_MODEL_NAME_RESPONSE_FIELD,
     STREAM_SSE_DATA_PREFIX,
     UNSAFE_PROXY_RESPONSE_HEADERS,
 )
@@ -2010,6 +2012,54 @@ class ProxyBaseLLMRequestProcessing:
                 return deployment
         return None
 
+    @staticmethod
+    def get_router_selected_model_name(
+        litellm_logging_obj: LiteLLMLoggingObj | None,
+    ) -> str | None:
+        """Model group an auto-routing strategy selected, or None if none fired.
+
+        The marker and ``deployment_model_name`` are written by different bucket
+        resolvers (``get_or_create_metadata_bucket`` vs
+        ``_get_router_metadata_variable_name``), so they can land in different
+        buckets on the same request. Resolve each across both.
+        """
+        litellm_params: Final = getattr(litellm_logging_obj, "litellm_params", None)
+        if not isinstance(litellm_params, dict):
+            return None
+        buckets: Final = tuple(
+            bucket for key in ("litellm_metadata", "metadata") if isinstance(bucket := litellm_params.get(key), dict)
+        )
+        if not any(bucket.get(AUTO_ROUTED_REQUEST_METADATA_KEY) is True for bucket in buckets):
+            return None
+        return next(
+            (
+                model_group
+                for bucket in buckets
+                if isinstance(model_group := bucket.get("deployment_model_name"), str) and model_group
+            ),
+            None,
+        )
+
+    @staticmethod
+    def set_router_selected_model_field(
+        *,
+        response_obj: object,
+        router_model_name: str | None,
+    ) -> None:
+        if not router_model_name:
+            return
+        if isinstance(response_obj, dict):
+            response_obj[ROUTER_MODEL_NAME_RESPONSE_FIELD] = router_model_name
+            return
+        try:
+            setattr(response_obj, ROUTER_MODEL_NAME_RESPONSE_FIELD, router_model_name)
+        except (AttributeError, TypeError, ValueError):
+            verbose_proxy_logger.debug(
+                "Could not set %s on response object of type %s",
+                ROUTER_MODEL_NAME_RESPONSE_FIELD,
+                type(response_obj),
+            )
+
     @staticmethod
     def _response_cost_from_logging_obj(
         *,
@@ -2508,6 +2558,10 @@ class ProxyBaseLLMRequestProcessing:
                 log_context=f"litellm_call_id={logging_obj.litellm_call_id}",
                 return_raw_model_name=_should_return_raw_model_name(self.data),
             )
+        self.set_router_selected_model_field(
+            response_obj=response,
+            router_model_name=self.get_router_selected_model_name(logging_obj),
+        )
 
         hidden_params = get_hidden_params_dict(response)  # get any updated response headers
         additional_headers = hidden_params.get("additional_headers", {}) or {}

litellm/proxy/db/db_spend_update_writer.py

@@ -316,6 +316,7 @@ class DBSpendUpdateWriter:
                 model_id=payload.get("model_id"),
                 llm_router=_get_llm_router,
                 cost_breakdown=metadata.get("cost_breakdown"),
+                recorded_autorouter_savings=metadata.get("autorouter_savings"),
             )
             transaction: Final = build_autorouter_turn_transaction(
                 payload=payload,
@@ -1877,6 +1878,7 @@ class DBSpendUpdateWriter:
                 llm_router=_get_llm_router,
                 usage_object=usage_obj,
                 cost_breakdown=_metadata.get("cost_breakdown"),
+                recorded_autorouter_savings=_metadata.get("autorouter_savings"),
             )
 
             daily_transaction: Final = BaseDailySpendTransaction(

litellm/proxy/db/spend_log_batching.py

@@ -10,10 +10,15 @@ hands the engine tens of megabytes in one statement and permanently costs
 hundreds of megabytes of RSS, which is what makes memory-based autoscaling
 read the wrong number.
 
-Bounding each statement by payload size instead caps that floor. Row-count
-batching alone cannot: the same 1000 rows range from well under a megabyte
-(spend counters only) to tens of megabytes (prompts stored), and only the
-byte budget tracks what the engine actually allocates.
+Bounding each statement caps that floor, and it takes two budgets because the
+engine charges for both terms. A byte budget is what tracks a prompt-carrying
+row, whose size swings by orders of magnitude, and a row budget is what tracks
+the engine's per-row bookkeeping, which a byte budget cannot see: rows holding
+attribution metadata only stay far under any useful byte budget, so it never
+binds and every statement runs at the caller's row cap. Measured on such a
+flush, the same 100,000 rows cost 151 MB of permanently resident engine RSS at
+1000 rows per statement against 25 MB at 100, with no statement anywhere near
+a 2 MB byte budget.
 """
 
 import json
@@ -99,16 +104,28 @@ def spend_log_queue_within_budget(
 def spend_log_write_batches(
     rows: Sequence[SpendLogRow],
     max_bytes: int,
+    max_rows: int,
 ) -> Iterator[Sequence[SpendLogRow]]:
-    """Yield consecutive slices of ``rows`` whose payload fits ``max_bytes``.
+    """Yield consecutive slices of ``rows`` within both ``max_bytes`` and ``max_rows``.
 
-    What is measured is the encoded slice, not the sum of its rows: rows become
-    one collection on the wire, so the brackets around them and the separator
-    between each pair count too. Summing rows alone under-states a slice by one
-    separator per row, which is negligible for prompt-carrying rows and is not
-    for a slice of many small ones, where the budget would be exceeded by the
-    row count. The two framing constants are derived from the serializer rather
-    than written down so they cannot drift from it.
+    What is measured for the byte budget is the encoded slice, not the sum of
+    its rows: rows become one collection on the wire, so the brackets around
+    them and the separator between each pair count too. Summing rows alone
+    under-states a slice by one separator per row, which is negligible for
+    prompt-carrying rows and is not for a slice of many small ones, where the
+    budget would be exceeded by the row count. The two framing constants are
+    derived from the serializer rather than written down so they cannot drift
+    from it.
+
+    Both budgets are needed because the engine's cost has two terms. Payload
+    bytes dominate when prompts are stored, and per-row bookkeeping dominates
+    when they are not: a slice of narrow rows costs the engine far more than
+    its bytes suggest, so a byte budget alone never binds on a deployment whose
+    rows carry no prompts and every statement stays at the caller's row cap.
+    Measured on a spend-log flush of rows carrying attribution metadata only,
+    writing the same 100,000 rows at 1000 rows per statement left 151 MB of
+    engine RSS resident against 25 MB at 100, with neither reaching a 2 MB byte
+    budget.
 
     Slices preserve input order and together cover every row exactly once. A
     row larger than ``max_bytes`` on its own is yielded alone rather than
@@ -120,7 +137,7 @@ def spend_log_write_batches(
     while start < len(rows):
         end = start + 1
         used = _STATEMENT_FRAMING_BYTES + sizes[start]
-        while end < len(rows) and used + _ROW_SEPARATOR_BYTES + sizes[end] <= max_bytes:
+        while end < len(rows) and end - start < max_rows and used + _ROW_SEPARATOR_BYTES + sizes[end] <= max_bytes:
             used += _ROW_SEPARATOR_BYTES + sizes[end]
             end += 1
         yield rows[start:end]

litellm/proxy/hooks/azure_content_safety.py

@@ -19,6 +19,8 @@ class _PROXY_AzureContentSafety(
 ):  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
 
+    enforces_request_content: bool = True
+
     def __init__(self, endpoint, api_key, thresholds=None):
         try:
             from azure.ai.contentsafety.aio import ContentSafetyClient

litellm/proxy/hooks/model_max_budget_limiter.py

@@ -1,21 +1,253 @@
 import json
+import time
+from collections.abc import Iterable, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
 from typing import Final
 
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import Span
+from litellm.litellm_core_utils.duration_parser import duration_in_seconds
+from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
 from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import (
-    BudgetConfig,
-    GenericBudgetConfigType,
-    StandardLoggingPayload,
-)
+from litellm.types.utils import BudgetConfig, StandardLoggingPayload
 
 VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX: Final = "virtual_key_spend"
 END_USER_SPEND_CACHE_KEY_PREFIX: Final = "end_user_model_spend"
+USER_SPEND_CACHE_KEY_PREFIX: Final = "user_model_spend"
+
+_SPEND_CACHE_KEY_PREFIXES: Final = MappingProxyType(
+    {
+        Litellm_EntityType.KEY: VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX,
+        Litellm_EntityType.USER: USER_SPEND_CACHE_KEY_PREFIX,
+        Litellm_EntityType.END_USER: END_USER_SPEND_CACHE_KEY_PREFIX,
+    }
+)
+
+_LEGACY_REQUEST_MODEL_SCOPES: Final = frozenset({Litellm_EntityType.KEY, Litellm_EntityType.END_USER})
+
+_PROCESS_STARTED_AT: Final = time.monotonic()
+
+_BUDGET_START_TIME_KEY_PREFIXES: Final = MappingProxyType(
+    {
+        Litellm_EntityType.KEY: "virtual_key_budget_start_time",
+        Litellm_EntityType.USER: "user_model_budget_start_time",
+        Litellm_EntityType.END_USER: "end_user_budget_start_time",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedModelBudget:
+    """The `model_max_budget` entry a request resolved to.
+
+    ``budget_model`` is the key as the operator configured it, not the model
+    name on the request. Every counter is keyed on it so enforcement, the
+    post-call increment and the `/key/info` + `/user/info` usage reads cannot
+    disagree about which counter a request belongs to.
+    """
+
+    budget_model: str
+    budget_config: BudgetConfig
+
+
+def model_budget_spend_cache_key(
+    entity_type: Litellm_EntityType,
+    entity_id: str | None,
+    budget_model: str,
+    budget_duration: str | None,
+) -> str:
+    """Sole owner of the per-model spend counter key, shared by its writer and all of its readers."""
+    return f"{_SPEND_CACHE_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
+
+
+def _legacy_request_model_spend_cache_key(
+    entity_type: Litellm_EntityType,
+    entity_id: str | None,
+    model: str,
+    resolved: ResolvedModelBudget,
+) -> str | None:
+    """The counter this request was billed to before the budget model owned the key, or None.
+
+    Upgrading proxies carry live counters keyed on the model as REQUESTED
+    (`openai/gpt-4`) rather than as configured (`gpt-4`), and those were the
+    counters the previous version enforced on. Nothing writes that spelling once
+    this version is running, so the pre-upgrade and post-upgrade counters hold
+    disjoint halves of one window and adding them is the window's real spend.
+
+    Only the key and end-user scopes ever had one. The user scope is introduced
+    by this change, so it has no counter to carry.
+
+    The carry stops one budget window after start-up, because a legacy counter
+    belongs to a window that was already open when this process replaced the one
+    writing it. Past that point the lookup could only ever miss.
+    """
+    budget_duration: Final = resolved.budget_config.budget_duration
+    if entity_type not in _LEGACY_REQUEST_MODEL_SCOPES or budget_duration is None:
+        return None
+    if time.monotonic() - _PROCESS_STARTED_AT >= duration_in_seconds(budget_duration):
+        return None
+    return model_budget_spend_cache_key(
+        entity_type=entity_type,
+        entity_id=entity_id,
+        budget_model=model,
+        budget_duration=budget_duration,
+    )
+
+
+def model_budget_start_time_cache_key(
+    entity_type: Litellm_EntityType,
+    entity_id: str | None,
+    budget_model: str,
+    budget_duration: str | None,
+) -> str:
+    """Window start for one (entity, budget model) pair.
+
+    Scoped per budget model because an entity may budget two models over
+    different periods, and a shared start time lets the shorter period restart
+    the longer one's window.
+    """
+    return f"{_BUDGET_START_TIME_KEY_PREFIXES[entity_type]}:{entity_id}:{budget_model}:{budget_duration}"
+
+
+def resolve_model_budget(model: str, model_max_budget: Mapping[str, object]) -> ResolvedModelBudget | None:
+    """Find the `model_max_budget` entry that governs `model`, or None."""
+    for candidate in _budget_model_candidates(model):
+        raw_budget_config = model_max_budget.get(candidate)
+        if raw_budget_config is None:
+            continue
+        if (budget_config := _usable_budget_config(raw_budget_config)) is None:
+            # An entry that will not validate cannot be keyed, so it cannot be
+            # enforced or incremented. Skip to the next candidate rather than
+            # raising: raising would abort every other scope's increment and turn
+            # a config typo into a 500, and stopping here would let one malformed
+            # specific entry disable a perfectly good bare-family budget beside
+            # it. The candidate chain already falls through an ABSENT entry, and
+            # an unparseable one is indistinguishable from absent to enforcement.
+            # `validate_model_max_budget` rejects these on the write path, so
+            # reaching here means config.yaml or a direct DB edit.
+            verbose_proxy_logger.warning(
+                "Ignoring unusable model_max_budget entry for %s; it cannot be enforced or tracked",
+                candidate,
+            )
+            continue
+        return ResolvedModelBudget(budget_model=candidate, budget_config=budget_config)
+    return None
+
+
+def _budget_model_candidates(model: str) -> tuple[str, ...]:
+    """Names a budget may be configured under for a request on `model`, most specific first.
+
+    Beyond the model as sent, a budget may be keyed on the model without its
+    ``{custom_llm_provider}/`` prefix (``gpt-4o`` governs ``openai/gpt-4o``), on
+    the Bedrock base model (``anthropic.claude-opus-4-8`` governs the
+    cross-region ``us.anthropic.claude-opus-4-8``), or on the bare family name
+    that Bedrock id shares with its direct-provider twin (``claude-opus-4-8``).
+    """
+    return tuple(dict.fromkeys((model, model.split("/")[-1], *_bedrock_candidates(model))))
+
+
+def _bedrock_candidates(model: str) -> tuple[str, ...]:
+    """Bedrock-only candidates, empty unless litellm prices `model` as a Bedrock model.
+
+    Gating on the cost map rather than on a vendor allowlist is what makes
+    splitting the leading dotted segment safe: most dotted model ids are not
+    Bedrock ids at all (``azure/gpt-4.1``, ``gpt-image-1.5``), and splitting one
+    of those would produce a garbage candidate.
+    """
+    base_model: Final = get_bedrock_base_model(model)
+    cost_entry: Final = litellm.model_cost.get(base_model)
+    if not isinstance(cost_entry, dict) or not str(cost_entry.get("litellm_provider", "")).startswith("bedrock"):
+        return ()
+    _, _, without_vendor = base_model.partition(".")
+    return (base_model, without_vendor) if without_vendor else (base_model,)
+
+
+async def build_model_max_budget_usage(
+    entity_type: Litellm_EntityType,
+    entity_id: str | None,
+    model_max_budget: Mapping[str, object] | None,
+    cache: DualCache | None,
+) -> dict[str, dict[str, object]]:
+    """Current-window spend per configured budget model, as `/key/info` and `/user/info` report it.
+
+    `cache` must be the DualCache the limiter writes the counters to; callers
+    read it off the limiter rather than re-deriving it, so a scope that is being
+    blocked can never report zero usage.
+    """
+    if cache is None or entity_id is None or not model_max_budget:
+        return {}
+
+    budgets: Final = tuple(
+        (budget_model, budget_config)
+        for budget_model, raw_budget_config in model_max_budget.items()
+        for budget_config in (_usable_budget_config(raw_budget_config),)
+        if budget_config is not None
+    )
+    if not budgets:
+        return {}
+    spend_keys: Final = tuple(
+        model_budget_spend_cache_key(
+            entity_type=entity_type,
+            entity_id=entity_id,
+            budget_model=budget_model,
+            budget_duration=budget_config.budget_duration,
+        )
+        for budget_model, budget_config in budgets
+    )
+    batched: Final = await cache.async_batch_get_cache(
+        keys=list(spend_keys)  # mutable-ok: async_batch_get_cache annotates keys as list, so one must exist here
+    )
+    # async_batch_get_cache returns None if it fails internally, and its result is
+    # index-aligned with `keys` otherwise. An unusable result reads as a miss,
+    # which is what a never-written counter already reads as.
+    current_spends: Final = (
+        tuple(batched) if isinstance(batched, list) and len(batched) == len(budgets) else (None,) * len(budgets)
+    )
+    return {
+        budget_model: {
+            "current_spend": round(_as_spend(current_spend), 4),
+            "budget_limit": budget_config.max_budget,
+            "time_period": budget_config.budget_duration,
+        }
+        for (budget_model, budget_config), current_spend in zip(budgets, current_spends, strict=True)
+    }
+
+
+def _usable_budget_config(raw_budget_config: object) -> BudgetConfig | None:
+    try:
+        budget_config: Final = BudgetConfig.model_validate(raw_budget_config)
+        if budget_config.budget_duration is None:
+            return None
+        duration_in_seconds(budget_config.budget_duration)
+    except Exception:  # noqa: BLE001  # a malformed entry must not fail the whole report
+        return None
+    return budget_config
+
+
+def _as_spend(current_spend: object) -> float:
+    try:
+        return float(current_spend or 0.0)  # pyright: ignore[reportArgumentType]  # non-numeric falls to the except
+    except (TypeError, ValueError):
+        return 0.0
+
+
+def _resolve_entity_model_budgets(
+    model: str,
+    entity_budgets: Iterable[tuple[Litellm_EntityType, str | None, object]],
+) -> tuple[tuple[Litellm_EntityType, str, ResolvedModelBudget], ...]:
+    """Drop the scopes that do not budget `model`, keeping only what can be incremented."""
+    return tuple(
+        (entity_type, entity_id, resolved)
+        for entity_type, entity_id, model_max_budget in entity_budgets
+        if entity_id is not None and isinstance(model_max_budget, Mapping) and model_max_budget
+        for resolved in (resolve_model_budget(model=model, model_max_budget=model_max_budget),)
+        if resolved is not None and resolved.budget_config.budget_duration is not None
+    )
 
 
 class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
@@ -41,47 +273,17 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
         Raises:
             BudgetExceededError: If the user_api_key_dict has exceeded the model budget
         """
-        _model_max_budget: Final = user_api_key_dict.model_max_budget
-        internal_model_max_budget: Final[GenericBudgetConfigType] = {}
-
-        for _model, _budget_info in _model_max_budget.items():
-            internal_model_max_budget[_model] = BudgetConfig(**_budget_info)
-
-        verbose_proxy_logger.debug(
-            "internal_model_max_budget %s",
-            json.dumps(internal_model_max_budget, indent=4, default=str),
+        return await self._is_entity_within_model_budget(
+            entity_type=Litellm_EntityType.KEY,
+            entity_id=user_api_key_dict.token,
+            model_max_budget=user_api_key_dict.model_max_budget,
+            model=model,
+            exceeded_message=(
+                f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, "
+                f"exceeded budget for model={model}"
+            ),
         )
 
-        # check if current model is in internal_model_max_budget
-        _current_model_budget_info: Final = self._get_request_model_budget_config(
-            model=model, internal_model_max_budget=internal_model_max_budget
-        )
-        if _current_model_budget_info is None:
-            verbose_proxy_logger.debug("Model %s not found in internal_model_max_budget", model)
-            return True
-
-        # check if current model is within budget
-        if _current_model_budget_info.max_budget and _current_model_budget_info.max_budget > 0:
-            _current_spend: Final = await self._get_virtual_key_spend_for_model(
-                user_api_key_hash=user_api_key_dict.token,
-                model=model,
-                key_budget_config=_current_model_budget_info,
-            )
-            if (
-                _current_spend is not None
-                and _current_model_budget_info.max_budget is not None
-                and _current_spend > _current_model_budget_info.max_budget
-            ):
-                raise litellm.BudgetExceededError(
-                    message=f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, exceeded budget for model={model}",
-                    current_cost=_current_spend,
-                    max_budget=_current_model_budget_info.max_budget,
-                    entity_type=Litellm_EntityType.KEY.value,
-                    entity_id=user_api_key_dict.token,
-                )
-
-        return True
-
     async def get_fallback_model_within_budget(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -96,10 +298,30 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
                 continue
         return None
 
+    async def is_user_within_model_budget(
+        self,
+        user_id: str,
+        user_model_max_budget: Mapping[str, object],
+        model: str,
+    ) -> bool:
+        """
+        Check if the internal user is within the model budget
+
+        Raises:
+            BudgetExceededError: If the user has exceeded the model budget
+        """
+        return await self._is_entity_within_model_budget(
+            entity_type=Litellm_EntityType.USER,
+            entity_id=user_id,
+            model_max_budget=user_model_max_budget,
+            model=model,
+            exceeded_message=f"LiteLLM User: {user_id}, exceeded budget for model={model}",
+        )
+
     async def is_end_user_within_model_budget(
         self,
         end_user_id: str,
-        end_user_model_max_budget: dict,
+        end_user_model_max_budget: Mapping[str, object],
         model: str,
     ) -> bool:
         """
@@ -108,116 +330,81 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
         Raises:
             BudgetExceededError: If the end_user has exceeded the model budget
         """
-        internal_model_max_budget: Final[GenericBudgetConfigType] = {}
-
-        for _model, _budget_info in end_user_model_max_budget.items():
-            internal_model_max_budget[_model] = BudgetConfig(**_budget_info)
-
-        verbose_proxy_logger.debug(
-            "end_user internal_model_max_budget %s",
-            json.dumps(internal_model_max_budget, indent=4, default=str),
+        return await self._is_entity_within_model_budget(
+            entity_type=Litellm_EntityType.END_USER,
+            entity_id=end_user_id,
+            model_max_budget=end_user_model_max_budget,
+            model=model,
+            exceeded_message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}",
         )
 
-        # check if current model is in internal_model_max_budget
-        _current_model_budget_info: Final = self._get_request_model_budget_config(
-            model=model, internal_model_max_budget=internal_model_max_budget
-        )
-        if _current_model_budget_info is None:
-            verbose_proxy_logger.debug("Model %s not found in end_user_model_max_budget", model)
+    async def _is_entity_within_model_budget(
+        self,
+        entity_type: Litellm_EntityType,
+        entity_id: str | None,
+        model_max_budget: Mapping[str, object] | None,
+        model: str,
+        exceeded_message: str,
+    ) -> bool:
+        if not model_max_budget:
+            return True
+        resolved: Final = resolve_model_budget(model=model, model_max_budget=model_max_budget)
+        if resolved is None:
+            verbose_proxy_logger.debug("Model %s not found in %s model_max_budget", model, entity_type.value)
             return True
 
-        # check if current model is within budget
-        if _current_model_budget_info.max_budget and _current_model_budget_info.max_budget > 0:
-            _current_spend: Final = await self._get_end_user_spend_for_model(
-                end_user_id=end_user_id,
-                model=model,
-                key_budget_config=_current_model_budget_info,
-            )
-            if (
-                _current_spend is not None
-                and _current_model_budget_info.max_budget is not None
-                and _current_spend > _current_model_budget_info.max_budget
-            ):
-                raise litellm.BudgetExceededError(
-                    message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}",
-                    current_cost=_current_spend,
-                    max_budget=_current_model_budget_info.max_budget,
-                    entity_type=Litellm_EntityType.END_USER.value,
-                    entity_id=end_user_id,
-                )
+        max_budget: Final = resolved.budget_config.max_budget
+        if max_budget is None or max_budget < 0:
+            return True
 
+        current_spend: Final = await self._get_spend_for_model_budget(
+            entity_type=entity_type,
+            entity_id=entity_id,
+            model=model,
+            resolved=resolved,
+        )
+        if current_spend >= max_budget:
+            raise litellm.BudgetExceededError(
+                message=exceeded_message,
+                current_cost=current_spend,
+                max_budget=max_budget,
+                entity_type=entity_type.value,
+                entity_id=entity_id,
+            )
         return True
 
-    async def _get_end_user_spend_for_model(
+    async def _get_spend_for_model_budget(
         self,
-        end_user_id: str,
+        entity_type: Litellm_EntityType,
+        entity_id: str | None,
         model: str,
-        key_budget_config: BudgetConfig,
-    ) -> float | None:
-        # 1. model: directly look up `model`
-        end_user_model_spend_cache_key = (
-            f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{key_budget_config.budget_duration}"
-        )
-        _current_spend = await self.dual_cache.async_get_cache(
-            key=end_user_model_spend_cache_key,
-        )
+        resolved: ResolvedModelBudget,
+    ) -> float:
+        """Spend charged to this budget in the current window, legacy counter included.
 
-        if _current_spend is None:
-            # 2. If 1, does not exist, check if passed as {custom_llm_provider}/model
-            end_user_model_spend_cache_key = f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{self._get_model_without_custom_llm_provider(model)}:{key_budget_config.budget_duration}"
-            _current_spend = await self.dual_cache.async_get_cache(
-                key=end_user_model_spend_cache_key,
-            )
-        return _current_spend
-
-    async def _get_virtual_key_spend_for_model(
-        self,
-        user_api_key_hash: str | None,
-        model: str,
-        key_budget_config: BudgetConfig,
-    ) -> float | None:
+        A counter that was never written is zero spend, not unknown spend. The
+        distinction only shows up at a zero-dollar cap, where skipping the
+        comparison would let the strictest possible limit admit every request.
         """
-        Get the current spend for a virtual key for a model
-
-        Lookup model in this order:
-            1. model: directly look up `model`
-            2. If 1, does not exist, check if passed as {custom_llm_provider}/model
-        """
-
-        # 1. model: directly look up `model`
-        virtual_key_model_spend_cache_key = (
-            f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{user_api_key_hash}:{model}:{key_budget_config.budget_duration}"
+        spend_key: Final = model_budget_spend_cache_key(
+            entity_type=entity_type,
+            entity_id=entity_id,
+            budget_model=resolved.budget_model,
+            budget_duration=resolved.budget_config.budget_duration,
         )
-        _current_spend = await self.dual_cache.async_get_cache(
-            key=virtual_key_model_spend_cache_key,
+        legacy_spend_key: Final = _legacy_request_model_spend_cache_key(
+            entity_type=entity_type,
+            entity_id=entity_id,
+            model=model,
+            resolved=resolved,
         )
+        current_spend: Final = _as_spend(await self._cached_spend(spend_key))
+        if legacy_spend_key is None or legacy_spend_key == spend_key:
+            return current_spend
+        return current_spend + _as_spend(await self._cached_spend(legacy_spend_key))
 
-        if _current_spend is None:
-            # 2. If 1, does not exist, check if passed as {custom_llm_provider}/model
-            # if "/" in model, remove first part before "/" - eg. openai/o1-preview -> o1-preview
-            virtual_key_model_spend_cache_key = f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{user_api_key_hash}:{self._get_model_without_custom_llm_provider(model)}:{key_budget_config.budget_duration}"
-            _current_spend = await self.dual_cache.async_get_cache(
-                key=virtual_key_model_spend_cache_key,
-            )
-        return _current_spend
-
-    def _get_request_model_budget_config(
-        self, model: str, internal_model_max_budget: GenericBudgetConfigType
-    ) -> BudgetConfig | None:
-        """
-        Get the budget config for the request model
-
-        1. Check if `model` is in `internal_model_max_budget`
-        2. If not, check if `model` without custom llm provider is in `internal_model_max_budget`
-        """
-        return internal_model_max_budget.get(model, None) or internal_model_max_budget.get(
-            self._get_model_without_custom_llm_provider(model), None
-        )
-
-    def _get_model_without_custom_llm_provider(self, model: str) -> str:
-        if "/" in model:
-            return model.split("/")[-1]
-        return model
+    async def _cached_spend(self, spend_key: str) -> float | None:
+        return await self.dual_cache.async_get_cache(key=spend_key)
 
     async def async_filter_deployments(
         self,
@@ -245,80 +432,63 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
 
         _litellm_params: Final[dict] = kwargs.get("litellm_params", {}) or {}
         _metadata: Final[dict] = _litellm_params.get("metadata", {}) or {}
-        user_api_key_model_max_budget: Final[dict | None] = _metadata.get("user_api_key_model_max_budget", None)
-        user_api_key_end_user_model_max_budget: Final[dict | None] = _metadata.get(
-            "user_api_key_end_user_model_max_budget", None
-        )
-        if (user_api_key_model_max_budget is None or len(user_api_key_model_max_budget) == 0) and (
-            user_api_key_end_user_model_max_budget is None or len(user_api_key_end_user_model_max_budget) == 0
-        ):
-            verbose_proxy_logger.debug(
-                "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event because user_api_key_model_max_budget and user_api_key_end_user_model_max_budget are None or empty."
-            )
-            return
+        payload_metadata: Final = standard_logging_payload.get("metadata") or {}
 
-        response_cost: Final[float] = standard_logging_payload.get("response_cost", 0)
         # Use model_group (the user-facing model alias, e.g. "gpt-4o") when
-        # available.  The enforcement path (is_key_within_model_budget) receives
-        # the model name from request_data["model"] which is the model group
-        # alias, so the spend tracking cache key must use the same name.
-        # Falling back to the deployment-level "model" field preserves
-        # behaviour for non-proxy or non-router deployments where model_group
-        # is None.
+        # available.  The enforcement path receives the model name from
+        # request_data["model"] which is the model group alias, so the spend
+        # tracking cache key must resolve from the same name.  Falling back to
+        # the deployment-level "model" field preserves behaviour for non-proxy
+        # or non-router deployments where model_group is None.
         model: Final = standard_logging_payload.get("model_group") or standard_logging_payload.get("model")
-        virtual_key: Final = standard_logging_payload.get("metadata", {}).get("user_api_key_hash")
-        end_user_id = standard_logging_payload.get("end_user") or standard_logging_payload.get("metadata", {}).get(
-            "user_api_key_end_user_id"
-        )
-
         if model is None:
             return
 
-        if (
-            virtual_key is not None
-            and user_api_key_model_max_budget is not None
-            and len(user_api_key_model_max_budget) > 0
-        ):
-            internal_model_max_budget: GenericBudgetConfigType = {}
-            for _model, _budget_info in user_api_key_model_max_budget.items():
-                internal_model_max_budget[_model] = BudgetConfig(**_budget_info)
-            key_budget_config = self._get_request_model_budget_config(
-                model=model, internal_model_max_budget=internal_model_max_budget
-            )
-            if key_budget_config is not None and key_budget_config.budget_duration:
-                virtual_spend_key: Final = (
-                    f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{key_budget_config.budget_duration}"
-                )
-                virtual_start_time_key: Final = f"virtual_key_budget_start_time:{virtual_key}"
-                await self._increment_spend_for_key(
-                    budget_config=key_budget_config,
-                    spend_key=virtual_spend_key,
-                    start_time_key=virtual_start_time_key,
-                    response_cost=response_cost,
-                )
+        response_cost: Final[float] = standard_logging_payload.get("response_cost", 0)
+        entity_budgets: Final = (
+            (
+                Litellm_EntityType.KEY,
+                payload_metadata.get("user_api_key_hash"),
+                _metadata.get("user_api_key_model_max_budget"),
+            ),
+            (
+                Litellm_EntityType.USER,
+                payload_metadata.get("user_api_key_user_id"),
+                _metadata.get("user_api_key_user_model_max_budget"),
+            ),
+            (
+                Litellm_EntityType.END_USER,
+                standard_logging_payload.get("end_user") or payload_metadata.get("user_api_key_end_user_id"),
+                _metadata.get("user_api_key_end_user_model_max_budget"),
+            ),
+        )
 
-        if (
-            end_user_id is not None
-            and user_api_key_end_user_model_max_budget is not None
-            and len(user_api_key_end_user_model_max_budget) > 0
-        ):
-            internal_model_max_budget: GenericBudgetConfigType = {}
-            for _model, _budget_info in user_api_key_end_user_model_max_budget.items():
-                internal_model_max_budget[_model] = BudgetConfig(**_budget_info)
-            key_budget_config = self._get_request_model_budget_config(
-                model=model, internal_model_max_budget=internal_model_max_budget
+        resolved_budgets: Final = _resolve_entity_model_budgets(model=model, entity_budgets=entity_budgets)
+        if not resolved_budgets:
+            verbose_proxy_logger.debug(
+                "Not running _PROXY_VirtualKeyModelMaxBudgetLimiter.async_log_success_event: "
+                "no key, user or end-user model_max_budget covers model=%s",
+                model,
+            )
+            return
+
+        for entity_type, entity_id, resolved in resolved_budgets:
+            await self._increment_spend_for_key(
+                budget_config=resolved.budget_config,
+                spend_key=model_budget_spend_cache_key(
+                    entity_type=entity_type,
+                    entity_id=entity_id,
+                    budget_model=resolved.budget_model,
+                    budget_duration=resolved.budget_config.budget_duration,
+                ),
+                start_time_key=model_budget_start_time_cache_key(
+                    entity_type=entity_type,
+                    entity_id=entity_id,
+                    budget_model=resolved.budget_model,
+                    budget_duration=resolved.budget_config.budget_duration,
+                ),
+                response_cost=response_cost,
             )
-            if key_budget_config is not None and key_budget_config.budget_duration:
-                end_user_spend_key: Final = (
-                    f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model}:{key_budget_config.budget_duration}"
-                )
-                end_user_start_time_key: Final = f"end_user_budget_start_time:{end_user_id}"
-                await self._increment_spend_for_key(
-                    budget_config=key_budget_config,
-                    spend_key=end_user_spend_key,
-                    start_time_key=end_user_start_time_key,
-                    response_cost=response_cost,
-                )
 
         if self.dual_cache.redis_cache is not None:
             await self._push_in_memory_increments_to_redis()

litellm/proxy/hooks/prompt_injection_detection.py

@@ -26,6 +26,8 @@ from litellm.utils import get_formatted_prompt
 
 
 class _OPTIONAL_PromptInjectionDetection(CustomLogger):
+    enforces_request_content: bool = True
+
     # Class variables or attributes
     def __init__(
         self,

litellm/proxy/litellm_pre_call_utils.py

@@ -64,6 +64,15 @@ _TRANSPORT_ONLY_CREDENTIAL_KEYS: Final = frozenset({"provider_specific_header",
 # Excludes the two explicit litellm headers which are handled with higher priority.
 _GENERIC_SESSION_ID_HEADER_RE: Final = re.compile(r"^x-.+-session-id$", re.IGNORECASE)
 _EXPLICIT_SESSION_HEADERS: Final = frozenset({"x-litellm-trace-id", "x-litellm-session-id"})
+# Codex carries its conversation uuid in unprefixed headers, so the
+# x-<vendor>-session-id convention above never matches it. Current builds send
+# ``session-id``/``thread-id``; builds before the codex-api split sent
+# ``session_id``/``conversation_id``. Ordered session before thread.
+_CODEX_SESSION_ID_HEADERS: Final = ("session-id", "session_id", "thread-id", "conversation_id")
+# Matches every first-party Codex originator: codex-tui, codex_cli_rs, codex_exec,
+# codex_vscode, "Codex ...". A separator is required so an unrelated "codexfoo" client
+# does not read as Codex.
+_CODEX_CLIENT_PREFIX_RE: Final = re.compile(r"^codex[-_ /]", re.IGNORECASE)
 # Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores
 # (covers UUIDs and most common session-id formats).
 _SESSION_ID_VALUE_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
@@ -583,6 +592,35 @@ def _extract_generic_session_id_from_headers(
     return None
 
 
+def _extract_codex_session_id_from_headers(
+    normalized: Mapping[str, str],
+) -> str | None:
+    """
+    Read Codex's conversation uuid off one of ``_CODEX_SESSION_ID_HEADERS``.
+
+    Codex sends no request metadata the Anthropic path could parse and no
+    ``x-``-prefixed session header, so without this every turn of a Codex session
+    falls through to a freshly generated per-call trace id and lands as its own
+    row in the logs instead of grouping.
+
+    Unprefixed names like ``session-id`` are generic enough that another client
+    could send one meaning something unrelated, and colliding values across
+    callers would merge their traces, so this only applies to callers that
+    identify as Codex.
+    """
+    user_agent: Final = normalized.get("user-agent")
+    if not isinstance(user_agent, str) or not is_codex_user_agent(user_agent):
+        return None
+    return next(
+        (
+            value
+            for value in (normalized.get(header) for header in _CODEX_SESSION_ID_HEADERS)
+            if isinstance(value, str) and _SESSION_ID_VALUE_RE.match(value)
+        ),
+        None,
+    )
+
+
 def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
     """
     Extract chain id for call chaining from request headers.
@@ -592,6 +630,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
     2. ``x-litellm-session-id`` (explicit)
     3. Any ``x-<vendor>-session-id`` header whose value looks like a session id
        (alphanumeric / UUID, at least 8 chars).  E.g. ``x-claude-code-session-id``.
+    4. Codex's unprefixed ``session-id`` / ``thread-id``, for Codex callers only.
 
     Header keys are matched case-insensitively so this works with raw header
     dicts from any transport.
@@ -606,6 +645,7 @@ def get_chain_id_from_headers(headers: dict[str, str] | None) -> str | None:
         normalized.get("x-litellm-trace-id")
         or normalized.get("x-litellm-session-id")
         or _extract_generic_session_id_from_headers(normalized)
+        or _extract_codex_session_id_from_headers(normalized)
     )
 
 
@@ -640,10 +680,13 @@ def is_claude_code_user_agent(user_agent: str) -> bool:
 
 
 def is_codex_user_agent(user_agent: str) -> bool:
-    """Codex identifies itself as ``codex_cli_rs/<version> ...`` (TUI),
-    ``codex_exec/<version> ...`` (exec mode), or ``codex_vscode/<version> ...``
-    (IDE extension); all share the ``codex_`` prefix."""
-    return user_agent.startswith("codex_")
+    """Codex builds its user agent as ``<originator>/<version> ...`` and ships
+    several first-party originators: ``codex-tui``, ``codex_cli_rs``,
+    ``codex_exec`` (exec mode), ``codex_vscode`` (IDE extension) and ``Codex ...``
+    (see ``is_first_party_originator`` in codex-rs). They agree only on the
+    ``codex`` stem, and the TUI sends a bare ``codex-tui`` with no version at all,
+    so match the stem plus a separator rather than any one spelling."""
+    return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent))
 
 
 def should_auto_drop_params_for_agentic_cli(user_agent: str, data: dict, proxy_config: ProxyConfig) -> bool:
@@ -1943,6 +1986,8 @@ async def add_litellm_data_to_request(
     # Follow same pattern as team and API key budgets
     data[_metadata_variable_name]["user_api_key_user_spend"] = user_api_key_dict.user_spend
     data[_metadata_variable_name]["user_api_key_user_max_budget"] = user_api_key_dict.user_max_budget
+    user_model_budget: Final = user_api_key_dict.user_model_max_budget
+    data[_metadata_variable_name]["user_api_key_user_model_max_budget"] = user_model_budget  # rebind-ok: out-param
 
     data[_metadata_variable_name]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata)
     data[_metadata_variable_name]["user_api_key_team_metadata"] = strip_callback_config(user_api_key_dict.team_metadata)

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -32,6 +32,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     object_permission_cache_key,
     user_object_permission_id_cache_key,
 )
+from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage
 from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks
 from litellm.proxy.management_endpoints.common_daily_activity import (
     DailySpendRecord,
@@ -817,6 +818,7 @@ def _build_user_info_response(
     keys: list[LiteLLM_VerificationToken] | None,
     team_list: list[TeamListResponseObject],
     teams_1: list[TeamListResponseObject] | None,
+    model_max_budget_usage: dict[str, dict[str, object]] | None = None,
 ) -> UserInfoResponse:
     """Create UserInfoResponse while filtering sensitive fields."""
     if user_info is None and keys is not None:
@@ -830,6 +832,8 @@ def _build_user_info_response(
     if isinstance(_user_info, dict):
         _user_info.pop("password", None)
         _user_info["metadata"] = _redact_scim_enterprise_metadata(_user_info.get("metadata"))
+        if model_max_budget_usage is not None:
+            _user_info["model_max_budget_usage"] = model_max_budget_usage
 
     return UserInfoResponse(
         user_id=user_id,
@@ -864,7 +868,7 @@ async def user_info(
     --header 'Authorization: Bearer sk-1234'
     ```
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.proxy_server import model_max_budget_limiter, prisma_client
 
     try:
         user_id = _normalize_user_info_user_id(request=request, user_id=user_id)
@@ -910,6 +914,12 @@ async def user_info(
             keys=keys,
             team_list=team_list,
             teams_1=teams_1,
+            model_max_budget_usage=await build_model_max_budget_usage(
+                entity_type=Litellm_EntityType.USER,
+                entity_id=user_id,
+                model_max_budget=getattr(user_info, "model_max_budget", None),
+                cache=model_max_budget_limiter.dual_cache,
+            ),
         )
 
         return response_data
@@ -1007,7 +1017,7 @@ async def user_info_v2(
     --header 'Authorization: Bearer sk-1234'
     ```
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.proxy_server import model_max_budget_limiter, prisma_client
 
     try:
         if prisma_client is None:
@@ -1062,6 +1072,13 @@ async def user_info_v2(
             sso_user_id=user_data.get("sso_user_id"),
             teams=user_data.get("teams") or [],
             object_permission=user_data.get("object_permission"),
+            model_max_budget=user_data.get("model_max_budget"),
+            model_max_budget_usage=await build_model_max_budget_usage(
+                entity_type=Litellm_EntityType.USER,
+                entity_id=user_data.get("user_id", user_id),
+                model_max_budget=user_data.get("model_max_budget"),
+                cache=model_max_budget_limiter.dual_cache,
+            ),
         )
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_info_v2(): Exception occured - %s", e)

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -29,6 +29,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, s
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
+from litellm.caching.dual_cache import DualCache
 from litellm.constants import (
     LENGTH_OF_LITELLM_GENERATED_KEY,
     LITELLM_PROXY_ADMIN_NAME,
@@ -47,7 +48,7 @@ from litellm.proxy._experimental.mcp_server.outbound_credentials.sso_assertion_s
     rotate_sso_identity_assertions_master_key,
 )
 from litellm.proxy._types import *
-from litellm.proxy._types import LiteLLM_VerificationToken, hash_token
+from litellm.proxy._types import Litellm_EntityType, LiteLLM_VerificationToken, hash_token
 from litellm.proxy.auth.auth_checks import (
     _delete_cache_key_object,
     can_team_access_model,
@@ -73,9 +74,7 @@ from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_k
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
-from litellm.proxy.hooks.model_max_budget_limiter import (
-    VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX,
-)
+from litellm.proxy.hooks.model_max_budget_limiter import build_model_max_budget_usage
 from litellm.proxy.management_endpoints.common_utils import (
     _check_passthrough_routes_caller_permission,
     _is_user_org_admin_for_team,
@@ -3511,62 +3510,17 @@ async def delete_key_fn(
         raise handle_exception_on_proxy(e)
 
 
-async def _get_model_max_budget_current_spend(
-    api_key_hash: str,
-    model: str,
-    budget_config: BudgetConfig,
-    user_api_key_cache: UserApiKeyCache,
-) -> float:
-    virtual_key_model_spend_cache_key = (
-        f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{api_key_hash}:{model}:{budget_config.budget_duration}"
-    )
-    current_spend: float | None = await user_api_key_cache.async_get_cache(
-        key=virtual_key_model_spend_cache_key,
-    )
-    if current_spend is None:
-        model_without_prefix: Final = model.split("/")[-1] if "/" in model else model
-        virtual_key_model_spend_cache_key = (
-            f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:"
-            f"{api_key_hash}:{model_without_prefix}:{budget_config.budget_duration}"
-        )
-        current_spend = await user_api_key_cache.async_get_cache(
-            key=virtual_key_model_spend_cache_key,
-        )
-    try:
-        return float(current_spend or 0.0)
-    except (TypeError, ValueError):
-        return 0.0
-
-
 async def _build_model_max_budget_usage(
     api_key_hash: str,
     model_max_budget: Mapping[str, Mapping[str, object]],
-    user_api_key_cache: UserApiKeyCache | None,
+    user_api_key_cache: DualCache | None,
 ) -> dict[str, dict[str, object]]:
-    if user_api_key_cache is None or not model_max_budget:
-        return {}
-
-    result: Final[dict[str, dict[str, object]]] = {}
-    for model, budget_info in model_max_budget.items():
-        try:
-            budget_config = BudgetConfig.model_validate(budget_info)
-            if budget_config.budget_duration is None:
-                continue
-            duration_in_seconds(budget_config.budget_duration)
-        except Exception:  # noqa: BLE001
-            continue
-        spend = await _get_model_max_budget_current_spend(
-            api_key_hash=api_key_hash,
-            model=model,
-            budget_config=budget_config,
-            user_api_key_cache=user_api_key_cache,
-        )
-        result[model] = {
-            "current_spend": round(spend, 4),
-            "budget_limit": budget_config.max_budget,
-            "time_period": budget_config.budget_duration,
-        }
-    return result
+    return await build_model_max_budget_usage(
+        entity_type=Litellm_EntityType.KEY,
+        entity_id=api_key_hash,
+        model_max_budget=model_max_budget,
+        cache=user_api_key_cache,
+    )
 
 
 @router.post(
@@ -3596,7 +3550,10 @@ async def info_key_fn_v2(
     -d {"keys": ["sk-1", "sk-2", "sk-3"]}
     ```
     """
-    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+    from litellm.proxy.proxy_server import (
+        model_max_budget_limiter,
+        prisma_client,
+    )
 
     try:
         if prisma_client is None:
@@ -3648,7 +3605,7 @@ async def info_key_fn_v2(
                 k_dict["model_max_budget_usage"] = await _build_model_max_budget_usage(
                     api_key_hash=k_token_hash,
                     model_max_budget=model_max_budget,
-                    user_api_key_cache=user_api_key_cache,
+                    user_api_key_cache=model_max_budget_limiter.dual_cache,
                 )
 
             filtered_key_info.append(k_dict)
@@ -3707,7 +3664,10 @@ async def info_key_fn(
 -H "Authorization: Bearer sk-test-example-key-123"
     ```
     """
-    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+    from litellm.proxy.proxy_server import (
+        model_max_budget_limiter,
+        prisma_client,
+    )
 
     try:
         if prisma_client is None:
@@ -3760,7 +3720,7 @@ async def info_key_fn(
             key_info["model_max_budget_usage"] = await _build_model_max_budget_usage(
                 api_key_hash=key_token_hash,
                 model_max_budget=model_max_budget,
-                user_api_key_cache=user_api_key_cache,
+                user_api_key_cache=model_max_budget_limiter.dual_cache,
             )
 
         # Attach object_permission if object_permission_id is set
@@ -3953,6 +3913,10 @@ async def generate_key_helper_fn(
         }
         if teams is not None:
             user_data["teams"] = teams
+        if model_max_budget:
+            # Only when supplied: the SSO and default-key callers reach this with the
+            # empty default, and writing that would clear an existing user's budgets.
+            user_data["model_max_budget"] = model_max_budget_json
         key_data: Final = {
             "token": token,
             "key_alias": key_alias,

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -312,6 +312,10 @@ def _validate_ptu_model_info(model_info: Mapping[str, object]) -> None:
     The rules live in litellm_core_utils.ptu_pricing so that config.yaml registration
     refuses the same deployments this endpoint does, for the same reason. Per-field bounds
     (positive count, non-negative rate) are enforced by ModelInfo itself.
+
+    Registration additionally requires an operator-declared ``model_info.id``, which this
+    endpoint does not: a stored deployment already holds a stable primary key, where a
+    config-declared one is otherwise keyed by a hash of its own parameters.
     """
     error: Final = ptu_config_error(model_info)
     if error is not None:

litellm/proxy/openai_files_endpoints/batch_guardrails.py

@@ -260,18 +260,24 @@ def _describe(custom_id: str | None) -> str:
     return f" (custom_id {safe})"
 
 
-def _iter_lines(source: BinaryIO) -> Iterator[tuple[int, str]]:
-    """Yield every non-blank line with its 1-based number, so both passes number records alike."""
+def _iter_lines(source: BinaryIO) -> Iterator[tuple[int, bytes]]:
+    """
+    Yield every non-blank line with its 1-based number, so both passes number records alike.
+
+    Bytes, not text. The upload validation immediately before this parses each line as bytes,
+    where the json module sniffs the encoding itself and accepts a leading byte order mark or a
+    lone surrogate. Decoding to `str` first is stricter than that, so a file written by any of
+    the editors that emit a BOM would pass validation and then fail the scan.
+    """
     for line_number, raw_line in enumerate(source, start=1):
-        text = raw_line.decode("utf-8")
-        if text.strip():
-            yield line_number, text
+        if raw_line.strip():
+            yield line_number, raw_line
 
 
 def _iter_records(source: BinaryIO) -> Iterator[_ParsedRecord]:
     """Yield one record per line, relying on the upload validation that already ran."""
-    for line_number, text in _iter_lines(source):
-        yield _ParsedRecord(line_number=line_number, payload=json.loads(text))
+    for line_number, raw_line in _iter_lines(source):
+        yield _ParsedRecord(line_number=line_number, payload=json.loads(raw_line))
 
 
 def _call_type_from_url(url: str) -> CallTypesLiteral | None:
@@ -282,7 +288,13 @@ def _call_type_from_url(url: str) -> CallTypesLiteral | None:
     ``/v1/responses`` in full would fall through to its body, where ``input`` reads as an
     embedding and the record gets scanned as the wrong call type rather than the right one.
     """
-    path: Final = urlsplit(url).path.split("?")[0].rstrip("/")
+    try:
+        path: Final = urlsplit(url).path.split("?")[0].rstrip("/")
+    except ValueError:
+        # urlsplit rejects a few malformed authorities outright, and the validation that ran
+        # before this only checks the key is present. An unreadable url is one we do not
+        # recognize, which is what falling back to the body shape already handles.
+        return None
     call_types: Final = get_call_types_for_route(path)
     if call_types is None:
         return None
@@ -308,8 +320,18 @@ def _scannable_call_type(url: object, body: Mapping[str, object]) -> CallTypesLi
 
 
 def _custom_id_of(payload: Mapping[str, object]) -> str | None:
+    """
+    The record's identifier, rendered as text.
+
+    The batch spec asks for a string, but callers do send numbers, and reporting those as null
+    would leave the one field a caller reconciles on empty for exactly the records it needs.
+    """
     custom_id: Final = payload.get("custom_id")
-    return custom_id if isinstance(custom_id, str) else None
+    if isinstance(custom_id, str):
+        # A lone surrogate parses out of the file but cannot be encoded back out, and this value
+        # is echoed in the response, so rendering it would fail the whole upload with a 500.
+        return custom_id.encode("utf-8", "replace").decode("utf-8")
+    return str(custom_id) if isinstance(custom_id, (int, float)) and not isinstance(custom_id, bool) else None
 
 
 def _fingerprint(body: Mapping[str, object], keys: frozenset[str]) -> str:
@@ -507,9 +529,9 @@ async def scan_batch_input_file(
     )
 
 
-def _read_spooled(redactions: BinaryIO, change: RecordRedacted) -> str:
+def _read_spooled(redactions: BinaryIO, change: RecordRedacted) -> bytes:
     redactions.seek(change.offset)
-    return redactions.read(change.length).decode("utf-8")
+    return redactions.read(change.length)
 
 
 def rewrite_batch_input_file(file_source: BinaryIO, result: BatchScanResult) -> BinaryIO:
@@ -532,12 +554,12 @@ def rewrite_batch_input_file(file_source: BinaryIO, result: BatchScanResult) ->
     )
     wrote_any = False  # rebind-ok: tracks whether a separator is needed
     try:
-        for line_number, text in _iter_lines(file_source):
+        for line_number, raw_line in _iter_lines(file_source):
             if line_number in dropped:
                 continue
             change = redacted.get(line_number)
-            line = text.rstrip("\n") if change is None else _read_spooled(result.redactions, change)
-            output.write((("\n" if wrote_any else "") + line).encode("utf-8"))
+            line = raw_line.rstrip(b"\n") if change is None else _read_spooled(result.redactions, change)
+            output.write(b"\n" + line if wrote_any else line)
             wrote_any = True
     except BaseException:
         output.close()

litellm/proxy/openai_files_endpoints/files_endpoints.py

@@ -145,25 +145,37 @@ async def _scan_batch_upload(
 
 
 def get_first_json_object(file_source: bytes | BinaryIO) -> dict | None:
+    """
+    The first record, used to pick a deployment when batch load balancing is on.
+
+    Read the way the upload validation reads it, since a file it accepted must not lose its
+    routing here: blank lines are not records and are skipped, and the line is parsed as bytes so
+    the json module sniffs the encoding rather than rejecting a leading byte order mark. Either
+    difference makes this return None, which silently sends the batch to the default provider.
+    """
     try:
         if isinstance(file_source, (bytes, bytearray)):
-            newline: Final = file_source.find(b"\n")
-            raw: Final = file_source if newline == -1 else file_source[:newline]
-            first_line = raw.decode("utf-8")
+            first_record: bytes | None = next((line for line in file_source.splitlines() if line.strip()), None)
         else:
+            # lazily, so a batch file that can be gigabytes is not read past its first record
             file_source.seek(0)
-            first_line = file_source.readline().decode("utf-8")
+            first_record = next((line for line in file_source if line.strip()), None)
             file_source.seek(0)
-        return json.loads(first_line.strip())
+        return None if first_record is None else json.loads(first_record.strip())
     except (json.JSONDecodeError, UnicodeDecodeError, OSError, ValueError):
         return None
 
 
 def get_model_from_json_obj(json_object: dict) -> str | None:
-    body: Final = json_object.get("body", {}) or {}
-    model: Final = body.get("model")
+    """
+    The model a record names, or None when it does not name one readably.
 
-    return model
+    The upload validation only checks that `body` is present, not that it is an object, so a
+    record can carry a string there and reach this. Returning None sends the upload down the
+    default-provider branch, which is what a record with no resolvable model already did.
+    """
+    body: Final = json_object.get("body")
+    return body.get("model") if isinstance(body, dict) else None
 
 
 async def _deprecated_loadbalanced_create_file(

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -64,6 +64,7 @@ from litellm.proxy._types import (
     ProxyException,
     UserAPIKeyAuth,
 )
+from litellm.proxy.auth.auth_utils import request_dispatched_to_pass_through_endpoint
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_request_processing import (
     ProxyBaseLLMRequestProcessing,
@@ -568,6 +569,22 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         _metadata["user_api_key"] = user_api_key_dict.api_key
         _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span
         _metadata["user_api_key_budget_reservation"] = user_api_key_dict.budget_reservation
+        # The per-model budget counters are keyed off these. get_sanitized_user_information_from_key
+        # returns StandardLoggingUserAPIKeyMetadata, which carries no budget field, so without this
+        # the post-call increment finds nothing and every passthrough request goes untracked and
+        # unenforced. Set after the client merge so a request body cannot supply its own budget.
+        #
+        # Only for the built-in provider routes. `get_model_from_request` returns
+        # None for a user-defined pass-through, deliberately: its body is forwarded
+        # verbatim, so `model` there names an UPSTREAM model rather than a
+        # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
+        # charging a counter anyway would track spend that nothing can refuse, and
+        # would attribute it to a budget the operator scoped to a LiteLLM model that
+        # merely shares the name.
+        if not request_dispatched_to_pass_through_endpoint(request):
+            _metadata["user_api_key_model_max_budget"] = user_api_key_dict.model_max_budget
+            _metadata["user_api_key_user_model_max_budget"] = user_api_key_dict.user_model_max_budget
+            _metadata["user_api_key_end_user_model_max_budget"] = user_api_key_dict.end_user_model_max_budget
         _metadata.update(
             LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
         )

litellm/proxy/proxy_server.py

@@ -247,6 +247,7 @@ from litellm.constants import (
     PROXY_BUDGET_RESCHEDULER_MAX_TIME,
     PROXY_BUDGET_RESCHEDULER_MIN_TIME,
     PROXY_CONFIG_RELOAD_INTERVAL_SECONDS,
+    ROUTER_MODEL_NAME_RESPONSE_FIELD,
     WEEKLY_SPEND_REPORT_JOB_ID,
 )
 from litellm.exceptions import RejectedRequestError
@@ -7913,6 +7914,10 @@ def _fast_serialize_simple_model_response_stream(
     for top_level_key in ("id", "object", "created"):
         if payload[top_level_key] is None:
             payload.pop(top_level_key)
+
+    router_model_name: Final = getattr(chunk, ROUTER_MODEL_NAME_RESPONSE_FIELD, None)
+    if router_model_name is not None:
+        payload[ROUTER_MODEL_NAME_RESPONSE_FIELD] = router_model_name
     return orjson.dumps(payload)
 
 
@@ -8210,6 +8215,9 @@ async def async_data_generator(
         model_mismatch_logged = False
         fallback_metadata_event_sent = False
         include_fallback_errors: Final = _should_include_fallback_errors(request_data)
+        # Fallbacks resolve on the first ``__anext__``, so the selected group is read
+        # per chunk off this object rather than snapshotted here.
+        router_logging_obj: Final = request_data.get("litellm_logging_obj")
         # Use a running string instead of list + join to avoid O(n^2) overhead.
         # Previously "".join(str_so_far_parts) was called every chunk, re-joining
         # the entire accumulated response. String += is O(n) amortized total.
@@ -8299,6 +8307,10 @@ async def async_data_generator(
                 fallback_was_attempted=fallback_was_attempted,
                 fallback_model_from_metadata=fallback_model_from_metadata,
             )
+            ProxyBaseLLMRequestProcessing.set_router_selected_model_field(
+                response_obj=chunk,
+                router_model_name=ProxyBaseLLMRequestProcessing.get_router_selected_model_name(router_logging_obj),
+            )
 
             if strip_stream_usage and _is_injected_stream_usage_artifact(chunk):
                 if pending_fallback_event:
@@ -8414,10 +8426,6 @@ async def async_data_generator(
         stream_completed = True
         yield f"data: {error_returned}\n\n"
     finally:
-        from litellm.proxy.common_request_processing import (
-            ProxyBaseLLMRequestProcessing,
-        )
-
         await ProxyBaseLLMRequestProcessing._finalize_streaming_generator_cleanup(
             request=request,
             request_data=request_data,

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -2726,6 +2726,34 @@
     ],
     "default_model_placeholder": "sap/gpt-4"
   },
+  {
+    "provider": "SCX_AI",
+    "provider_display_name": "SCX.ai",
+    "litellm_provider": "scx-ai",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://api.scx.ai/v1",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "scx-ai/GLM-5.2"
+  },
   {
     "provider": "Snowflake",
     "provider_display_name": "Snowflake",

litellm/proxy/spend_tracking/ptu_flat_cost_rollup.py

@@ -324,7 +324,6 @@ class _LoadedDeployments:
 
     models: tuple[PTUModel, ...]
     scanned_ids: frozenset[str]
-    config_sourced: bool
 
 
 def _running_router() -> object | None:
@@ -371,7 +370,6 @@ async def _load_ptu_models(prisma_client: "PrismaClient") -> _LoadedDeployments:
     )
     return _LoadedDeployments(
         models=models,
-        config_sourced=bool(config_records),
         scanned_ids=db_ids
         | frozenset(record.model_id for record in config_records)
         | frozenset(model.model_id for model in models),
@@ -385,9 +383,11 @@ async def run_ptu_flat_cost_rollup(
 ) -> RollupResult:
     """Rollup one UTC day of flat PTU cost across all PTU-configured model deployments.
 
-    Defaults to yesterday UTC. Authoritative for the day: it upserts the current charges
-    first, then deletes the day's sentinel rows this run did not refresh, so a
-    since-removed, invalidated, or now-out-of-window deployment leaves no stale charge.
+    Defaults to yesterday UTC. It upserts the current charges first, then deletes the
+    day's sentinel rows it scanned and did not refresh, so an invalidated or
+    now-out-of-window deployment leaves no stale charge. A deployment it cannot see is
+    left alone, since its charge records capacity that was reserved and this run has no
+    grounds to retract it.
 
     The prune predicate is ``updated_at < run_started`` rather than "not in the charge
     set I computed", which matters under concurrency: whether a row is garbage becomes a
@@ -436,7 +436,7 @@ async def run_ptu_flat_cost_rollup(
             prisma_client,
             date_str=date_str,
             run_started=run_started,
-            scanned_ids=loaded.scanned_ids if loaded.config_sourced else None,
+            scanned_ids=loaded.scanned_ids,
         )
 
     verbose_proxy_logger.info(
@@ -724,8 +724,8 @@ async def _deliver_alert(alert: "Callable[[str], Awaitable[None]] | None", messa
         verbose_proxy_logger.error("PTU rollup: could not deliver the failed-charge alert: %s", exc)
 
 
-def _prune_filter(*, date_str: str, cutoff: datetime, chunk: "tuple[str, ...] | None") -> "Mapping[str, object]":
-    """One delete statement's predicate. An absent chunk leaves the sweep unbounded.
+def _prune_filter(*, date_str: str, cutoff: datetime, chunk: "tuple[str, ...]") -> "Mapping[str, object]":
+    """One delete statement's predicate, bounded to the deployments in ``chunk``.
 
     Returns a plain dict because the query builder serialises the mapping it is handed and
     rejects a read-only view of one.
@@ -734,7 +734,7 @@ def _prune_filter(*, date_str: str, cutoff: datetime, chunk: "tuple[str, ...] |
         "date": date_str,
         "api_key": PTU_SENTINEL_API_KEY,
         "updated_at": {"lt": cutoff},  # mutable-ok: prisma comparison filter
-        **({} if chunk is None else {"model": {"in": chunk}}),  # mutable-ok: prisma membership filter
+        "model": {"in": chunk},  # mutable-ok: prisma membership filter
     }
 
 
@@ -743,7 +743,7 @@ async def _prune_unrefreshed_sentinel_rows(
     *,
     date_str: str,
     run_started: datetime,
-    scanned_ids: frozenset[str] | None,
+    scanned_ids: frozenset[str],
 ) -> None:
     """Delete the day's PTU sentinel rows this run looked at and did not refresh.
 
@@ -754,25 +754,22 @@ async def _prune_unrefreshed_sentinel_rows(
     different hosts, and the grace separates a row that is hours old from one written
     seconds ago without waiting on clocks agreeing.
 
-    A run that priced a deployment only its own host declares must also name the
-    deployments it scanned. Staleness alone is sufficient while every run derives its
-    charges from the same table, because then any two runs compute the same set, so a
-    database-only run still sweeps by timestamp exactly as it always has. Once one host's
-    charges come from a file the others cannot read, a row it never considered is not
-    evidence of anything, and deleting it drops a charge that host is responsible for.
+    It must also be a deployment this run could see. A charge already written is a record
+    of capacity that was reserved, so the only rows a run may retract are the ones it can
+    reassess: a deployment it scanned and then declined to charge, because the window
+    closed or the PTU config was removed. A row whose deployment is absent from every
+    source the run reads is not evidence that the reservation never happened, only that
+    this host cannot account for it. A deployment the router refused to register is in that
+    same bucket as one that was removed, because neither reaches the scan.
 
-    Where the bound applies the ids go out in chunks, because each is one bind variable and
-    the server rejects a statement carrying more than 32767 of them, which a proxy holding
-    that many deployments would otherwise hit every night with no handler above here.
+    The ids go out in chunks, because each is one bind variable and the server rejects a
+    statement carrying more than 32767 of them, which a proxy holding that many
+    deployments would otherwise hit every night with no handler above here.
     """
     cutoff: Final = run_started - timedelta(seconds=PTU_PRUNE_SKEW_GRACE_SECONDS)
-    ordered: Final = () if scanned_ids is None else tuple(sorted(scanned_ids))
-    chunks: Final = (
-        (None,)
-        if scanned_ids is None
-        else tuple(
-            ordered[start : start + _PRUNE_ID_CHUNK_SIZE] for start in range(0, len(ordered), _PRUNE_ID_CHUNK_SIZE)
-        )
+    ordered: Final = tuple(sorted(scanned_ids))
+    chunks: Final = tuple(
+        ordered[start : start + _PRUNE_ID_CHUNK_SIZE] for start in range(0, len(ordered), _PRUNE_ID_CHUNK_SIZE)
     )
     filters: Final = tuple(_prune_filter(date_str=date_str, cutoff=cutoff, chunk=chunk) for chunk in chunks)
     deletions: Final = tuple(
@@ -781,10 +778,10 @@ async def _prune_unrefreshed_sentinel_rows(
     deleted: Final = sum(deletions)
     if deleted:
         verbose_proxy_logger.info(
-            "PTU rollup for %s: pruned %s stale sentinel row(s) across %s deployment(s)",
+            "PTU rollup for %s: pruned %s stale sentinel row(s) of %s deployment(s) considered",
             date_str,
             deleted,
-            "every" if scanned_ids is None else len(scanned_ids),
+            len(ordered),
         )
 
 

litellm/proxy/spend_tracking/savings.py

@@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final, NamedTuple
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY
 from litellm.litellm_core_utils.llm_cost_calc.utils import _get_cost_per_unit, generic_cost_per_token
 
 if TYPE_CHECKING:
@@ -437,6 +438,97 @@ def extract_cache_creation_tokens(usage_object: Mapping[str, object] | None) ->
     return int(written)
 
 
+def _proxy_llm_router() -> "Router | None":
+    """The running proxy's router, or ``None`` outside a proxy (public rates only)."""
+    try:
+        from litellm.proxy.proxy_server import llm_router
+    except Exception:  # noqa: BLE001  # SDK-only usage has no proxy module to import
+        return None
+    return llm_router
+
+
+def _numeric_savings(value: object) -> float | None:
+    """``value`` as a recorded savings figure, or ``None`` when it is not one."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return float(value)
+
+
+def autorouter_savings_for_request(
+    model: str | None,
+    custom_llm_provider: str | None,
+    routing_decision: Mapping[str, object] | None,
+    usage_object: Mapping[str, object] | None,
+    model_id: str | None = None,
+    llm_router: "Callable[[], Router | None] | None" = None,
+    cost_breakdown: Mapping[str, object] | None = None,
+) -> float | None:
+    """Auto-router savings for one request, or ``None`` when the driver is off.
+
+    ``None`` and ``0.0`` are different facts: ``None`` means this request cannot carry a
+    figure at all (no routing decision, no baseline, unusable usage), while ``0.0`` is a
+    real figure for a routed request whose baseline resolved to the served deployment.
+    Never raises: pricing failures inside degrade to zero, and the driver-off cases
+    return ``None``, so this is safe on the logging path where a raise would fail the
+    request's logging.
+    """
+    usage: Final = _usage_from_spend_log(usage_object)
+    if usage is None or not model:
+        return None
+    # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline
+    # the deciding router recorded on its decision; neither means the driver is off.
+    decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {}
+    recorded: Final = decision.get("savings_baseline_model")
+    recorded_id: Final = decision.get("savings_baseline_deployment_id")
+    configured: Final = litellm.autorouter_savings_baseline_model
+    baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None)
+    baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None
+    if not decision or not baseline_model:
+        return None
+    router_instance: Final = llm_router() if llm_router else None
+    return compute_autorouter_savings(
+        baseline_model=baseline_model,
+        selected_model=model,
+        selected_provider=custom_llm_provider,
+        usage=usage,
+        # Absent means the router never recorded a shape, which is the conservative
+        # reading: charge the cache write rather than claim a first turn's saving.
+        conversation_continuing=decision.get("conversation_continuing") is not False,
+        selected_info=_effective_model_info(router_instance, model_id, model or ""),
+        baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
+        cost_breakdown=cost_breakdown,
+    )
+
+
+def autorouter_savings_for_logging_payload(
+    request_metadata: Mapping[str, object],
+    model: str | None,
+    custom_llm_provider: str | None,
+    model_id: str | None,
+    usage_object: Mapping[str, object] | None,
+    cost_breakdown: Mapping[str, object] | None,
+) -> float | None:
+    """The figure the logging payload records for a request, or ``None`` when none should be.
+
+    Internal sub-calls (the auto-router classifier, shadow eval's shadow and judge legs)
+    are excluded here for the same reason the spend writer zeroes them: they can carry a
+    real routing decision, but they are not requests the caller made, so a figure stamped
+    on them would report savings for traffic no user sent.
+    """
+    if request_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY):
+        return None
+    routing_decision: Final = request_metadata.get("routing_decision")
+    return autorouter_savings_for_request(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        routing_decision=routing_decision if isinstance(routing_decision, Mapping) else None,
+        usage_object=usage_object,
+        model_id=model_id,
+        llm_router=_proxy_llm_router,
+        cost_breakdown=cost_breakdown,
+    )
+
+
 def compute_savings_spend(
     model: str | None,
     custom_llm_provider: str | None,
@@ -446,6 +538,7 @@ def compute_savings_spend(
     model_id: str | None = None,
     llm_router: "Callable[[], Router | None] | None" = None,
     cost_breakdown: Mapping[str, object] | None = None,
+    recorded_autorouter_savings: object = None,
 ) -> SavingsSpend:
     """
     Dollar savings for one request, split by optimization driver.
@@ -488,6 +581,11 @@ def compute_savings_spend(
     hypothetical token delta off flat rate keys, so they are blind to tiered pricing in
     the same way; that is pre-existing behaviour on two shipped drivers rather than
     something introduced here, and moving those numbers is its own change.
+
+    ``recorded_autorouter_savings`` is the figure the logging path stamped on the spend
+    log's metadata, honoured over recomputation so the rollup, the turn table and the
+    per-request record cannot disagree; rows written before the field shipped carry
+    nothing and recompute, mirroring ``_recorded_token_cost``.
     """
     # Deployment rates when the request came through one, public rates otherwise --
     # `_effective_model_info` merges a deployment's configured prices over the built-in
@@ -505,32 +603,24 @@ def compute_savings_spend(
     write_premium: Final = max(cache_creation_input_tokens, 0) * (cache_write_cost - input_cost)
     prompt_caching: Final = read_discount - write_premium
 
-    usage: Final = _usage_from_spend_log(usage_object)
-    if usage is None or not model:
-        return SavingsSpend(compression=compression, prompt_caching=prompt_caching)
-
-    # The configured `autorouter_savings_baseline_model` wins; otherwise the baseline
-    # the deciding router recorded on its decision; neither means the driver is off.
-    decision: Final = routing_decision if isinstance(routing_decision, Mapping) else {}
-    recorded: Final = decision.get("savings_baseline_model")
-    recorded_id: Final = decision.get("savings_baseline_deployment_id")
-    configured: Final = litellm.autorouter_savings_baseline_model
-    baseline_model: Final = configured or (recorded if isinstance(recorded, str) else None)
-    baseline_id: Final = recorded_id if configured is None and isinstance(recorded_id, str) else None
+    # The figure the logging path recorded wins, before the usage gate on purpose: a row
+    # whose usage no longer parses still carries the number computed when it did.
+    recorded_savings: Final = _numeric_savings(recorded_autorouter_savings)
     autorouter: Final = (
-        compute_autorouter_savings(
-            baseline_model=baseline_model,
-            selected_model=model,
-            selected_provider=custom_llm_provider,
-            usage=usage,
-            # Absent means the router never recorded a shape, which is the conservative
-            # reading: charge the cache write rather than claim a first turn's saving.
-            conversation_continuing=decision.get("conversation_continuing") is not False,
-            selected_info=_effective_model_info(router_instance, model_id, model or ""),
-            baseline_info=_effective_model_info(router_instance, baseline_id, baseline_model or ""),
+        recorded_savings
+        if recorded_savings is not None
+        else autorouter_savings_for_request(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            routing_decision=routing_decision,
+            usage_object=usage_object,
+            model_id=model_id,
+            llm_router=llm_router,
             cost_breakdown=cost_breakdown,
         )
-        if decision and baseline_model
-        else 0.0
     )
-    return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter)
+    return SavingsSpend(
+        compression=compression,
+        prompt_caching=prompt_caching,
+        autorouter=0.0 if autorouter is None else autorouter,
+    )

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -100,6 +100,7 @@ def _get_spend_logs_metadata(
     litellm_overhead_time_ms: float | None = None,
     cost_breakdown: CostBreakdown | None = None,
     litellm_call_id: str | None = None,
+    autorouter_savings: float | None = None,
 ) -> SpendLogsMetadata:
     if metadata is None:
         return SpendLogsMetadata(
@@ -132,6 +133,7 @@ def _get_spend_logs_metadata(
             max_retries=None,
             cost_breakdown=None,
             compression_savings=None,
+            autorouter_savings=autorouter_savings,
             litellm_call_id=litellm_call_id,
         )
     verbose_proxy_logger.debug(
@@ -158,6 +160,7 @@ def _get_spend_logs_metadata(
     clean_metadata["cold_storage_object_key"] = cold_storage_object_key
     clean_metadata["litellm_overhead_time_ms"] = litellm_overhead_time_ms
     clean_metadata["cost_breakdown"] = cost_breakdown
+    clean_metadata["autorouter_savings"] = autorouter_savings
     clean_metadata["litellm_call_id"] = litellm_call_id
 
     return clean_metadata
@@ -385,6 +388,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
         cost_breakdown=(
             standard_logging_payload.get("cost_breakdown", None) if standard_logging_payload is not None else None
         ),
+        autorouter_savings=(
+            standard_logging_payload.get("autorouter_savings", None) if standard_logging_payload is not None else None
+        ),
         litellm_call_id=cast(
             str | None,
             kwargs.get("litellm_call_id") or litellm_params.get("litellm_call_id"),

litellm/proxy/utils.py

@@ -11,7 +11,7 @@ import sys
 import threading
 import time
 import traceback
-from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping, Sequence
+from collections.abc import AsyncGenerator, Awaitable, Callable, Collection, Coroutine, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import date, datetime, timedelta, timezone
 from email.mime.multipart import MIMEMultipart
@@ -28,6 +28,7 @@ from litellm.constants import (
     MAX_TEAM_LIST_LIMIT,
     SPEND_LOG_QUEUE_MAX_BYTES,
     SPEND_LOG_WRITE_BATCH_MAX_BYTES,
+    SPEND_LOG_WRITE_BATCH_MAX_ROWS,
 )
 from litellm.proxy._types import (
     CommonProxyErrors,
@@ -185,6 +186,7 @@ if TYPE_CHECKING:
     from litellm.models.team import LiteLLM_TeamTableCachedObj
     from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
     from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
+    from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
 
     Span = _Span | object
 else:
@@ -407,6 +409,46 @@ def _exception_changes_request_flow(exc: BaseException) -> bool:
     return isinstance(exc, (SensitiveDataRouteException, ModifyResponseException))
 
 
+def _policy_state_metadata(data: Mapping[str, object]) -> Mapping[str, object]:
+    """
+    Return the metadata bucket the policy engine wrote its pipeline state into.
+
+    The route decides the bucket (``litellm_metadata`` for ``/v1/messages``,
+    responses, batches, files and bedrock, ``metadata`` everywhere else), and both
+    buckets can be present at once because callers send their own provider-facing
+    ``metadata`` (Claude Code sends ``metadata.user_id``) or their own
+    ``litellm_metadata``. Pipeline slots are stripped from caller input before the
+    policy engine runs, so whichever bucket carries them is the proxy's own write.
+    """
+    return next(
+        (
+            bucket
+            for bucket in (data.get("metadata"), data.get("litellm_metadata"))
+            if isinstance(bucket, dict)
+            and ("_guardrail_pipelines" in bucket or "_pipeline_managed_guardrails" in bucket)
+        ),
+        {},
+    )
+
+
+def _policy_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]:
+    pipelines: Final = _policy_state_metadata(data).get("_guardrail_pipelines")
+    return (
+        tuple(cast("Sequence[tuple[str, GuardrailPipeline]]", pipelines))  # cast-ok: the policy engine wrote the slot
+        if pipelines
+        else ()
+    )
+
+
+def _pipeline_managed_guardrail_names(data: Mapping[str, object]) -> frozenset[str]:
+    managed: Final = _policy_state_metadata(data).get("_pipeline_managed_guardrails")
+    return (
+        frozenset(cast("Collection[str]", managed))  # cast-ok: the policy engine wrote these guardrail names
+        if managed
+        else frozenset()
+    )
+
+
 def _prompt_block_text(block: object) -> str:
     if isinstance(block, str):
         return block
@@ -563,6 +605,7 @@ class _CallbackCapabilities:
     has_streaming_chunk_override: bool = False
     has_guardrail: bool = False
     has_pre_call_override: bool = False
+    has_content_enforcer: bool = False
     # Tuple[(resolved_callback, "override" | "apply_guardrail"), ...]
     # Ordered the same as ``litellm.callbacks``; used to build the streaming
     # iterator chain without re-scanning per request.
@@ -1444,8 +1487,7 @@ class ProxyLogging:
 
         Returns the (possibly modified) data dict.
         """
-        metadata: Final = data.get("metadata", data.get("litellm_metadata", {})) or {}
-        pipelines: Final = metadata.get("_guardrail_pipelines")
+        pipelines: Final = _policy_pipelines(data)
         if not pipelines:
             return data
 
@@ -1529,19 +1571,26 @@ class ProxyLogging:
 
     def has_pre_call_guardrails(self, request_metadata: Mapping[str, object]) -> bool:
         """
-        Whether any guardrail or guardrail pipeline would inspect a request carrying this metadata.
+        Whether anything configured would inspect the content of a request carrying this metadata.
 
         Evaluated with the same predicate the pre-call loop uses, so a proxy configured only with
         post-call guardrails answers False. Callers that must pay a real cost to build the hook's
         input, such as streaming a batch input file off disk, use this to skip that work.
+
+        A content-enforcing ``CustomLogger`` counts too. It is not a guardrail and has no event
+        hook to consult, but it judges the payload the same way, so a proxy configured only with
+        one of those still has something to say about every record.
         """
         if request_metadata.get("_guardrail_pipelines"):
             return True
+        caps: Final = ProxyLogging._callback_capabilities()
+        if caps.has_content_enforcer:
+            return True
         probe: Final = {"metadata": dict(request_metadata)}  # mutable-ok: should_run_guardrail takes a dict
         return any(
             isinstance(callback, CustomGuardrail)
             and callback.should_run_guardrail(data=probe, event_type=GuardrailEventHooks.pre_call)
-            for callback in ProxyLogging._callback_capabilities().resolved_callbacks
+            for callback in caps.resolved_callbacks
         )
 
     # The actual implementation of the function
@@ -1622,8 +1671,7 @@ class ProxyLogging:
             )
 
             # Get pipeline-managed guardrails to skip in normal loop
-            metadata: Final = data.get("metadata", data.get("litellm_metadata", {})) or {}
-            pipeline_managed: Final[set] = metadata.get("_pipeline_managed_guardrails", set())
+            pipeline_managed: Final = _pipeline_managed_guardrail_names(data)
 
             caps: Final = ProxyLogging._callback_capabilities()
             # Skip the per-request callback walk entirely when nothing in
@@ -1631,7 +1679,11 @@ class ProxyLogging:
             # CustomGuardrail is configured. Saves the loop overhead +
             # ``time.time()`` x2 per registered callback for the common
             # "callbacks=[]" case on small / dev deployments.
-            if not caps.has_guardrail and (guardrails_only or not caps.has_pre_call_override):
+            if (
+                not caps.has_guardrail
+                and not caps.has_content_enforcer
+                and (guardrails_only or not caps.has_pre_call_override)
+            ):
                 if data is not None:
                     self._process_guardrail_metadata(data)
                 return data
@@ -1668,9 +1720,9 @@ class ProxyLogging:
                         data = result
 
                     elif (
-                        not guardrails_only
-                        and _callback is not None
+                        _callback is not None
                         and isinstance(_callback, CustomLogger)
+                        and (not guardrails_only or _callback.enforces_request_content)
                         and "async_pre_call_hook" in vars(_callback.__class__)
                         and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook
                     ):
@@ -1922,6 +1974,7 @@ class ProxyLogging:
         has_streaming_chunk_override = False
         has_guardrail = False
         has_pre_call_override = False
+        has_content_enforcer = False
         iterator_overrides: Final[list[tuple[Any, str]]] = []  # (callback, kind)
         resolved_callbacks: Final[list[CustomLogger]] = []
 
@@ -1973,6 +2026,8 @@ class ProxyLogging:
                 has_streaming_chunk_override = True
             if "async_pre_call_hook" in cls_attrs:
                 has_pre_call_override = True
+                if resolved.enforces_request_content is True:
+                    has_content_enforcer = True
 
         caps: Final = _CallbackCapabilities(
             has_post_call_response_headers=has_post_call_response_headers,
@@ -1981,6 +2036,7 @@ class ProxyLogging:
             has_streaming_chunk_override=has_streaming_chunk_override,
             has_guardrail=has_guardrail,
             has_pre_call_override=has_pre_call_override,
+            has_content_enforcer=has_content_enforcer,
             iterator_overrides=tuple(iterator_overrides),
             resolved_callbacks=tuple(resolved_callbacks),
         )
@@ -6048,7 +6104,9 @@ class ProxyUpdateSpend:
                             batch_with_dates = [prisma_client.jsonify_object({**entry}) for entry in batch]
                             isolation_budget = MAX_SPEND_LOG_ISOLATION_FAILURES_PER_BATCH
                             for statement_rows in spend_log_write_batches(
-                                batch_with_dates, SPEND_LOG_WRITE_BATCH_MAX_BYTES
+                                batch_with_dates,
+                                SPEND_LOG_WRITE_BATCH_MAX_BYTES,
+                                SPEND_LOG_WRITE_BATCH_MAX_ROWS,
                             ):
                                 isolation_budget = await _create_spend_logs_with_poison_isolation(
                                     SpendLogsRepository(prisma_client),

litellm/router.py

@@ -44,6 +44,7 @@ from litellm.caching.caching import (
     RedisClusterCache,
 )
 from litellm.constants import (
+    AUTO_ROUTED_REQUEST_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
     DEFAULT_HEALTH_CHECK_INTERVAL,
@@ -67,6 +68,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.ptu_pricing import (
     is_ptu_cost_attribution_enabled,
     ptu_config_error,
+    ptu_identity_error,
+    ptu_terms,
     zeroed_ptu_pricing,
 )
 from litellm.litellm_core_utils.request_timeout_resolver import (
@@ -7694,6 +7697,9 @@ class Router:
         _model_name: str,
         _litellm_params: dict,
         _model_info: dict,
+        *,
+        declared_id: str | None = None,
+        duplicate_ids: frozenset[str] = frozenset(),
     ) -> Deployment | None:
         """
         Create a deployment object and add it to the model list
@@ -7706,7 +7712,19 @@ class Router:
         """
         try:
             config_sourced: Final = _model_info.get("db_model") is not True
-            ptu_error: Final = ptu_config_error(_model_info, model_name=_model_name) if config_sourced else None
+            identity_error: Final = (
+                ptu_identity_error(
+                    declared_id=declared_id,
+                    taken=declared_id in duplicate_ids,
+                    current_id=_model_info.get("id"),
+                    model_name=_model_name,
+                )
+                if config_sourced and ptu_terms(_model_info) is not None
+                else None
+            )
+            ptu_error: Final = (
+                (ptu_config_error(_model_info, model_name=_model_name) or identity_error) if config_sourced else None
+            )
             if ptu_error is not None and is_ptu_cost_attribution_enabled():
                 raise ValueError(ptu_error)
             zeroed_pricing: Final = zeroed_ptu_pricing(_model_info, _litellm_params) if config_sourced else None
@@ -8209,6 +8227,13 @@ class Router:
         self._invalidate_access_groups_cache()
         # we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
 
+        declared_ids: Final = tuple(
+            str(entry["model_info"]["id"])
+            for entry in original_model_list
+            if isinstance(entry.get("model_info"), dict) and entry["model_info"].get("id") is not None
+        )
+        duplicate_ids: Final = frozenset(model_id for model_id in declared_ids if declared_ids.count(model_id) > 1)
+
         for model in original_model_list:
             _model_name = model.pop("model_name")
             _litellm_params = model.pop("litellm_params")
@@ -8220,6 +8245,8 @@ class Router:
 
             _model_info: dict = model.pop("model_info", {})
 
+            declared_id = None if _model_info.get("id") is None else str(_model_info["id"])
+
             # check if model info has id
             if "id" not in _model_info:
                 _id = self.generate_model_id(_model_name, _litellm_params)
@@ -8235,6 +8262,8 @@ class Router:
                         _model_name=_model_name,
                         _litellm_params=_litellm_params,
                         _model_info=_model_info,
+                        declared_id=declared_id,
+                        duplicate_ids=duplicate_ids,
                     )
             else:
                 self._create_deployment(
@@ -8242,6 +8271,8 @@ class Router:
                     _model_name=_model_name,
                     _litellm_params=_litellm_params,
                     _model_info=_model_info,
+                    declared_id=declared_id,
+                    duplicate_ids=duplicate_ids,
                 )
 
         verbose_router_logger.debug("\nInitialized Model List %s", self.get_model_names())
@@ -9153,10 +9184,27 @@ class Router:
 
         ## SET MODEL TO 'model=' - if base_model is None + not azure
         if custom_llm_provider == "azure" and base_model is None:
-            verbose_router_logger.error(
-                "Could not identify azure model '%s'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models",
-                _model,
+            # Router init auto-registers every deployment name into
+            # litellm.model_cost as a zeroed stub, so membership alone can't
+            # tell a resolvable name apart; require usable limits/costs.
+            _azure_fallback_key = _model if _model.startswith("azure/") else f"azure/{_model}"
+            _fallback_entry = litellm.model_cost.get(_azure_fallback_key)
+            _fallback_resolves = _fallback_entry is not None and (
+                (_fallback_entry.get("max_input_tokens") or 0) > 0
+                or (_fallback_entry.get("max_tokens") or 0) > 0
+                or (_fallback_entry.get("input_cost_per_token") or 0) > 0
             )
+            if _fallback_resolves:
+                verbose_router_logger.debug(
+                    "Azure deployment '%s' has no base_model set; using '%s' from the model cost map for max tokens, cost tracking, etc.",
+                    _model,
+                    _azure_fallback_key,
+                )
+            else:
+                verbose_router_logger.error(
+                    "Could not identify azure model '%s'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models",
+                    _model,
+                )
         elif custom_llm_provider != "azure":
             model = _model
 
@@ -11647,6 +11695,9 @@ class Router:
             self._stamp_or_clear_metadata_key(
                 request_kwargs=request_kwargs, key=CONSUMED_REQUEST_TAGS_METADATA_KEY, value=None
             )
+            self._stamp_or_clear_metadata_key(
+                request_kwargs=request_kwargs, key=AUTO_ROUTED_REQUEST_METADATA_KEY, value=None
+            )
             return None
 
         pre_routing_hook_response: Final = await selected_strategy.strategy.async_pre_routing_hook(
@@ -11674,6 +11725,13 @@ class Router:
                 request_tags=_get_tags_from_request_kwargs(request_kwargs),
             ),
         )
+        # Gates the proxy's `router_model_name` response field; the body `model` is
+        # always restamped back to the alias the client sent.
+        self._stamp_or_clear_metadata_key(
+            request_kwargs=request_kwargs,
+            key=AUTO_ROUTED_REQUEST_METADATA_KEY,
+            value=(True if pre_routing_hook_response is not None else None),
+        )
 
         # `model` (the alias, e.g. "smart-router") is never the deployment actually
         # called - apply the router marker's own litellm_params to the request,

litellm/types/llms/anthropic.py

@@ -683,7 +683,7 @@ ANTHROPIC_API_ONLY_HEADERS: Final = {  # fails if calling anthropic on vertex ai
 
 
 class AnthropicThinkingParam(TypedDict, total=False):
-    type: Literal["enabled", "adaptive"]
+    type: ReadOnly[Literal["enabled", "adaptive", "disabled"]]
     budget_tokens: int
 
 

litellm/types/utils.py

@@ -154,6 +154,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
     supports_web_search: bool | None
     supports_reasoning: bool | None
     supports_adaptive_thinking: bool | None
+    thinking_always_on: ReadOnly[bool | None]
     supports_tool_search: bool | None
     supports_mid_conversation_system: bool | None
     supports_url_context: bool | None
@@ -3192,6 +3193,7 @@ class StandardLoggingPayload(TypedDict):
     stream: bool | None
     response_cost: float
     cost_breakdown: CostBreakdown | None  # Detailed cost breakdown
+    autorouter_savings: ReadOnly[float | None]  # None = not an auto-routed caller request; 0.0 is a real figure
     response_cost_failure_debug_info: StandardLoggingModelCostFailureDebugInformation | None
     status: StandardLoggingPayloadStatus
     status_fields: StandardLoggingPayloadStatusFields
@@ -3788,6 +3790,7 @@ class LlmProviders(str, Enum):
     LIBERTAI = "libertai"
     PINSTRIPES = "pinstripes"
     COGNITION = "cognition"
+    SCX_AI = "scx-ai"
     DARKBLOOM = "darkbloom"
     META = "meta"
     LITELLM_AGENT = "litellm_agent"

litellm/utils.py

@@ -5753,6 +5753,7 @@ def _get_model_info_helper(
                 supports_url_context=_model_info.get("supports_url_context", None),
                 supports_reasoning=_model_info.get("supports_reasoning", None),
                 supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
+                thinking_always_on=_model_info.get("thinking_always_on", None),
                 supports_tool_search=_model_info.get("supports_tool_search", None),
                 supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
                 supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),

model_prices_and_context_window.json

@@ -1232,6 +1232,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
+        "thinking_always_on": true,
         "supports_function_calling": true,
         "supports_vision": true,
         "supports_prompt_caching": false,
@@ -1404,6 +1405,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1440,6 +1442,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1476,6 +1479,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -1512,6 +1516,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -3021,6 +3026,7 @@
         "cache_creation_input_token_cost_above_1hr": 2e-05,
         "cache_read_input_token_cost": 1e-06,
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -4875,6 +4881,38 @@
         "supports_tool_choice": true,
         "supports_vision": false
     },
+    "azure/gpt-audio-mini": {
+        "deprecation_date": "2027-04-06",
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "azure",
+        "max_input_tokens": 128000,
+        "max_output_tokens": 16384,
+        "max_tokens": 16384,
+        "mode": "chat",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_function_calling": true,
+        "supports_native_streaming": true,
+        "supports_parallel_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
     "azure/gpt-audio-mini-2025-10-06": {
         "deprecation_date": "2027-04-06",
         "input_cost_per_audio_token": 1e-05,
@@ -5088,6 +5126,38 @@
         "supports_system_messages": true,
         "supports_tool_choice": true
     },
+    "azure/gpt-realtime-mini": {
+        "cache_creation_input_audio_token_cost": 3e-07,
+        "cache_read_input_token_cost": 6e-08,
+        "input_cost_per_audio_token": 1e-05,
+        "input_cost_per_image": 8e-07,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "azure",
+        "max_input_tokens": 32000,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "realtime",
+        "output_cost_per_audio_token": 2e-05,
+        "output_cost_per_token": 2.4e-06,
+        "supported_endpoints": [
+            "/v1/realtime"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "audio"
+        ],
+        "supports_audio_input": true,
+        "supports_audio_output": true,
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true
+    },
     "azure/gpt-realtime-mini-2025-10-06": {
         "cache_creation_input_audio_token_cost": 3e-07,
         "cache_read_input_token_cost": 6e-08,
@@ -12780,6 +12850,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -19491,106 +19562,6 @@
         },
         "web_search_billing_unit": "per_query"
     },
-    "gemini-3.1-flash-lite-image": {
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "litellm_provider": "vertex_ai-language-models",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": false,
-        "supports_prompt_caching": true,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
-    "gemini/gemini-3.1-flash-lite-image": {
-        "rpm": 1000,
-        "tpm": 4000000,
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "input_cost_per_token_batches": 1.25e-07,
-        "litellm_provider": "gemini",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "output_cost_per_token_batches": 7.5e-07,
-        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": true,
-        "supports_prompt_caching": false,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
-    "vertex_ai/gemini-3.1-flash-lite-image": {
-        "input_cost_per_image": 0.00028,
-        "input_cost_per_token": 2.5e-07,
-        "litellm_provider": "vertex_ai-language-models",
-        "max_input_tokens": 65536,
-        "max_output_tokens": 4096,
-        "max_tokens": 4096,
-        "mode": "image_generation",
-        "output_cost_per_image": 0.0336,
-        "output_cost_per_image_token": 3e-05,
-        "output_cost_per_token": 1.5e-06,
-        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models",
-        "supported_endpoints": [
-            "/v1/chat/completions",
-            "/v1/completions",
-            "/v1/batch"
-        ],
-        "supported_modalities": [
-            "text",
-            "image"
-        ],
-        "supported_output_modalities": [
-            "text",
-            "image"
-        ],
-        "supports_function_calling": false,
-        "supports_prompt_caching": true,
-        "supports_response_schema": false,
-        "supports_reasoning": true,
-        "supports_system_messages": true,
-        "supports_vision": true
-    },
     "gemini-3.1-flash-image": {
         "deprecation_date": "2027-05-28",
         "input_cost_per_image": 0.00056,
@@ -19668,6 +19639,44 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-3.1-flash-lite-image": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "gemini-3.1-flash-lite-preview": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_audio_token": 5e-07,
@@ -21498,6 +21507,42 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-3.1-flash-lite-image": {
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "rpm": 1000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite-image",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_vision": true,
+        "tpm": 4000000
+    },
     "gemini/deep-research-pro-preview-12-2025": {
         "input_cost_per_image": 0.0011,
         "input_cost_per_token": 2e-06,
@@ -26034,33 +26079,33 @@
         "supports_minimal_reasoning_effort": true
     },
     "gpt-5.6": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
-        "cache_creation_input_token_cost_flex": 3.125e-06,
-        "cache_creation_input_token_cost_priority": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
-        "cache_read_input_token_cost_flex": 2.5e-07,
-        "cache_read_input_token_cost_priority": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
-        "input_cost_per_token_above_272k_tokens_flex": 5e-06,
-        "input_cost_per_token_batches": 2.5e-06,
-        "input_cost_per_token_flex": 2.5e-06,
-        "input_cost_per_token_priority": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
+        "cache_creation_input_token_cost_flex": 2.5e-06,
+        "cache_creation_input_token_cost_priority": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
+        "cache_read_input_token_cost_flex": 2e-07,
+        "cache_read_input_token_cost_priority": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
+        "input_cost_per_token_above_272k_tokens_flex": 4e-06,
+        "input_cost_per_token_batches": 2e-06,
+        "input_cost_per_token_flex": 2e-06,
+        "input_cost_per_token_priority": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
-        "output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
-        "output_cost_per_token_batches": 1.5e-05,
-        "output_cost_per_token_flex": 1.5e-05,
-        "output_cost_per_token_priority": 6e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
+        "output_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_flex": 1e-05,
+        "output_cost_per_token_priority": 4e-05,
         "regional_processing_uplift_multiplier_eu": 1.1,
         "regional_processing_uplift_multiplier_us": 1.1,
         "search_context_cost_per_query": {
@@ -26097,33 +26142,33 @@
         "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-sol": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_creation_input_token_cost_above_272k_tokens_flex": 6.25e-06,
-        "cache_creation_input_token_cost_flex": 3.125e-06,
-        "cache_creation_input_token_cost_priority": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "cache_read_input_token_cost_above_272k_tokens_flex": 5e-07,
-        "cache_read_input_token_cost_flex": 2.5e-07,
-        "cache_read_input_token_cost_priority": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
-        "input_cost_per_token_above_272k_tokens_flex": 5e-06,
-        "input_cost_per_token_batches": 2.5e-06,
-        "input_cost_per_token_flex": 2.5e-06,
-        "input_cost_per_token_priority": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 5e-06,
+        "cache_creation_input_token_cost_flex": 2.5e-06,
+        "cache_creation_input_token_cost_priority": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 4e-07,
+        "cache_read_input_token_cost_flex": 2e-07,
+        "cache_read_input_token_cost_priority": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
+        "input_cost_per_token_above_272k_tokens_flex": 4e-06,
+        "input_cost_per_token_batches": 2e-06,
+        "input_cost_per_token_flex": 2e-06,
+        "input_cost_per_token_priority": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
-        "output_cost_per_token_above_272k_tokens_flex": 2.25e-05,
-        "output_cost_per_token_batches": 1.5e-05,
-        "output_cost_per_token_flex": 1.5e-05,
-        "output_cost_per_token_priority": 6e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 1.5e-05,
+        "output_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_flex": 1e-05,
+        "output_cost_per_token_priority": 4e-05,
         "regional_processing_uplift_multiplier_eu": 1.1,
         "regional_processing_uplift_multiplier_us": 1.1,
         "search_context_cost_per_query": {
@@ -26365,19 +26410,19 @@
         "supports_parallel_function_calling": true
     },
     "daybreak-blue-latest": {
-        "cache_creation_input_token_cost": 6.25e-06,
-        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
-        "cache_read_input_token_cost": 5e-07,
-        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
-        "input_cost_per_token": 5e-06,
-        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost": 5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
+        "cache_read_input_token_cost": 4e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 8e-07,
+        "input_cost_per_token": 4e-06,
+        "input_cost_per_token_above_272k_tokens": 8e-06,
         "litellm_provider": "openai",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "chat",
-        "output_cost_per_token": 3e-05,
-        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "output_cost_per_token": 2e-05,
+        "output_cost_per_token_above_272k_tokens": 3e-05,
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/responses"
@@ -31140,6 +31185,23 @@
         "supports_video_input": true,
         "supports_vision": true
     },
+    "moonshot/kimi-k3": {
+        "cache_read_input_token_cost": 3e-07,
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "moonshot",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://platform.kimi.ai/docs/pricing/chat-k3",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "moonshot/kimi-latest": {
         "cache_read_input_token_cost": 1.5e-07,
         "deprecation_date": "2026-01-28",
@@ -36724,6 +36786,40 @@
         "supports_vision": true,
         "source": "https://cloud.sambanova.ai/plans/pricing"
     },
+    "scx-ai/GLM-5.2": {
+        "cache_read_input_token_cost": 2.2e-07,
+        "input_cost_per_token": 6.1e-07,
+        "litellm_provider": "scx-ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.98e-06,
+        "source": "https://scx.ai/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": false
+    },
+    "scx-ai/Qwen3.8-Max": {
+        "cache_read_input_token_cost": 2.1e-07,
+        "input_cost_per_token": 1.65e-06,
+        "litellm_provider": "scx-ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 4.99e-06,
+        "source": "https://scx.ai/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "snowflake/claude-3-5-sonnet": {
         "litellm_provider": "snowflake",
         "max_input_tokens": 200000,
@@ -40365,6 +40461,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -40398,6 +40495,7 @@
             "search_context_size_medium": 0.01
         },
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -41006,6 +41104,44 @@
         "supports_reasoning": false,
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models"
     },
+    "vertex_ai/gemini-3.1-flash-lite-image": {
+        "cache_read_input_token_cost": 2.5e-08,
+        "input_cost_per_image": 0.00028,
+        "input_cost_per_token": 2.5e-07,
+        "input_cost_per_token_batches": 1.25e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 65536,
+        "max_output_tokens": 4096,
+        "max_tokens": 4096,
+        "mode": "image_generation",
+        "output_cost_per_image": 0.0336,
+        "output_cost_per_image_token": 3e-05,
+        "output_cost_per_token": 1.5e-06,
+        "output_cost_per_token_batches": 7.5e-07,
+        "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "image"
+        ],
+        "supports_function_calling": false,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": false,
+        "supports_response_schema": false,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "vertex_ai/gemini-3.1-flash-lite-preview": {
         "cache_read_input_token_cost": 2.5e-08,
         "input_cost_per_audio_token": 5e-07,
@@ -48545,6 +48681,156 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "us.openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5.5e-06,
+        "input_cost_per_token_above_272k_tokens": 1.1e-05,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.375e-05,
+        "cache_read_input_token_cost": 5.5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
+        "output_cost_per_token": 3.3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_token_above_272k_tokens": 1e-05,
+        "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 1.25e-05,
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 1e-06,
+        "output_cost_per_token": 3e-05,
+        "output_cost_per_token_above_272k_tokens": 4.5e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "us.openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2.2e-06,
+        "input_cost_per_token_above_272k_tokens": 4.4e-06,
+        "cache_creation_input_token_cost": 2.75e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 5.5e-06,
+        "cache_read_input_token_cost": 2.2e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
+        "output_cost_per_token": 1.32e-05,
+        "output_cost_per_token_above_272k_tokens": 1.98e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_272k_tokens": 4e-06,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "cache_creation_input_token_cost_above_272k_tokens": 5e-06,
+        "cache_read_input_token_cost": 2e-07,
+        "cache_read_input_token_cost_above_272k_tokens": 4e-07,
+        "output_cost_per_token": 1.2e-05,
+        "output_cost_per_token_above_272k_tokens": 1.8e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "us.openai.gpt-5.6-luna": {
+        "input_cost_per_token": 2.2e-07,
+        "input_cost_per_token_above_272k_tokens": 4.4e-07,
+        "cache_creation_input_token_cost": 2.75e-07,
+        "cache_creation_input_token_cost_above_272k_tokens": 5.5e-07,
+        "cache_read_input_token_cost": 2.2e-08,
+        "cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
+        "output_cost_per_token": 1.32e-06,
+        "output_cost_per_token_above_272k_tokens": 1.98e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "global.openai.gpt-5.6-luna": {
+        "input_cost_per_token": 2e-07,
+        "input_cost_per_token_above_272k_tokens": 4e-07,
+        "cache_creation_input_token_cost": 2.5e-07,
+        "cache_creation_input_token_cost_above_272k_tokens": 5e-07,
+        "cache_read_input_token_cost": 2e-08,
+        "cache_read_input_token_cost_above_272k_tokens": 4e-08,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_272k_tokens": 1.8e-06,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "bedrock_mantle/openai.gpt-5.5": {
         "input_cost_per_token": 5.5e-06,
         "cache_read_input_token_cost": 5.5e-07,
@@ -49754,6 +50040,7 @@
         },
         "source": "https://docs.claude.com/en/docs/about-claude/models/overview",
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_mid_conversation_system": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
@@ -49789,6 +50076,7 @@
         },
         "source": "https://docs.claude.com/en/docs/about-claude/models/overview",
         "supports_adaptive_thinking": true,
+        "thinking_always_on": true,
         "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
@@ -49967,6 +50255,14 @@
                     "supports_adaptive_thinking": true
                 }
             },
+            {
+                "name": "claude-always-on-thinking",
+                "pattern": "claude-(?:fable|mythos)-",
+                "description": "Any Claude Fable or Mythos id, under any provider namespace and any version. These families always think and reject thinking.type=disabled with a 400; the Anthropic transformations omit the param instead, so the model falls back to its default adaptive thinking.",
+                "model_info": {
+                    "thinking_always_on": true
+                }
+            },
             {
                 "name": "claude-mid-conversation-system",
                 "pattern": "claude-[a-z]+-(?:4[-._](?:[89]|[1-9]\\d)(?!\\d)|(?:[5-9]|[1-9]\\d)(?!\\d)(?:[-._]\\d{1,2}(?!\\d))?)",

model_prices_and_context_window.schema.json

@@ -706,6 +706,9 @@
         "supports_xhigh_reasoning_effort": {
           "type": "boolean"
         },
+        "thinking_always_on": {
+          "type": "boolean"
+        },
         "tiered_pricing": {
           "type": "array",
           "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.",

provider_endpoints_support.json

@@ -2261,6 +2261,23 @@
         "interactions": true
       }
     },
+    "scx-ai": {
+      "display_name": "SCX.ai (`scx-ai`)",
+      "url": "https://docs.litellm.ai/docs/providers/scx_ai",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": false,
+        "responses": false,
+        "embeddings": false,
+        "image_generations": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "moderations": false,
+        "batches": false,
+        "rerank": false,
+        "a2a": false
+      }
+    },
     "snowflake": {
       "display_name": "Snowflake (`snowflake`)",
       "url": "https://docs.litellm.ai/docs/providers/snowflake",

pyproject.toml

@@ -341,9 +341,13 @@ filterwarnings = [
 paths_to_mutate = [
     "litellm/proxy/management_endpoints/",
 ]
+# Only the unit tier that maps to paths_to_mutate. mutmut times and
+# coverage-maps this whole set once before mutating, so a tier that needs a
+# seeded database (tests/proxy_behavior/) kills the run before it starts, and
+# a mutation score is only meaningful against the tests that claim to cover
+# the mutated code anyway.
 tests_dir = [
     "tests/test_litellm/proxy/management_endpoints/",
-    "tests/proxy_behavior/management/",
 ]
 also_copy = [
     "litellm/",
@@ -360,10 +364,16 @@ mutate_only_covered_lines = true
 # - rerunning a "failed" test on a mutant would mask which mutants are killed
 #   vs. survive, so reruns are wrong for mutation testing regardless.
 # - xdist is unnecessary inside mutmut (mutmut handles its own parallelism).
+# test_saml_sso.py cannot run inside mutmut's mutants/ sandbox: the copied tree
+# re-imports cryptography's hash classes under a second identity, so x509 .sign()
+# rejects the SHA256 instance the fixture builds with "Algorithm must be a
+# registered hash algorithm". Nothing to do with mutation coverage, and one
+# erroring test is enough to end the stats phase before any mutant runs.
 pytest_add_cli_args = [
     "-p", "no:retry",
     "-p", "no:rerunfailures",
     "-p", "no:xdist",
+    "--ignore=tests/test_litellm/proxy/management_endpoints/test_saml_sso.py",
 ]
 
 [tool.coverage.run]

ruff-tests.toml

@@ -26,6 +26,16 @@
 # PT014   the same `parametrize` case listed twice. The copy re-runs an assertion that
 #         already passed and adds no coverage, and it usually marks a case someone meant
 #         to vary and forgot to edit
+# F811    a name bound twice where the first binding was never used. Mostly a repeated
+#         import, but the same rule is what catches a second `def test_x` silently
+#         replacing the first, and a local that shadows an import the module still calls
+# PT017   an `assert` on the caught error inside `except`. Nothing runs the handler when
+#         the call stops raising, so the test goes green on the exact regression it was
+#         written to catch. `pytest.raises` fails when the call succeeds
+# RUF043  a `match=` pattern carrying regex metacharacters in a plain string. `match=` is
+#         `re.search`, so a `.` copied out of an error message is a wildcard and the block
+#         accepts messages the author never meant to accept. Mark a real regex raw, wrap a
+#         literal message in `re.escape`, and the pattern says which one it is
 #
 # No target-version here on purpose: it resolves from requires-python (>=3.10), so
 # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that
@@ -33,4 +43,19 @@
 
 line-length = 120
 
-lint.select = ["F821", "B011", "B015", "B017", "B018", "PT011", "PT012", "PT014", "PT015", "PLR0133", "PLW0127"]
+lint.select = [
+    "F811",
+    "F821",
+    "B011",
+    "B015",
+    "B017",
+    "B018",
+    "PT011",
+    "PT012",
+    "PT014",
+    "PT015",
+    "PT017",
+    "PLR0133",
+    "PLW0127",
+    "RUF043",
+]

scripts/mutation_report.py

@@ -22,6 +22,7 @@ import tomllib
 from collections import defaultdict
 from difflib import SequenceMatcher
 from pathlib import Path
+from typing import Final, NamedTuple
 from textwrap import dedent
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -33,16 +34,24 @@ def load_mutmut_config() -> dict:
         return tomllib.load(f)["tool"]["mutmut"]
 
 
-def get_survivors() -> list[str]:
+class MutmutResults(NamedTuple):
+    survivors: tuple[str, ...]
+    reported: int
+
+
+def get_survivors() -> MutmutResults:
     proc = subprocess.run(
         [*MUTMUT_INVOCATION, "results"], capture_output=True, text=True, check=False
     )
-    survivors = []
-    for line in proc.stdout.splitlines():
-        m = re.match(r"\s*(\S+):\s*survived\s*$", line)
-        if m:
-            survivors.append(m.group(1))
-    return survivors
+    verdicts = tuple(
+        m.groups()
+        for line in proc.stdout.splitlines()
+        if (m := re.match(r"\s*(\S+):\s*(\S.*?)\s*$", line))
+    )
+    return MutmutResults(
+        survivors=tuple(name for name, verdict in verdicts if verdict == "survived"),
+        reported=len(verdicts),
+    )
 
 
 def get_mutmut_show(mutant_name: str) -> str:
@@ -222,7 +231,52 @@ def render_meta_style_mutant(
     return "\n".join(out)
 
 
-def render(config: dict, survivors: list[str], stats: dict | None) -> str:
+RESOLVED_KEYS: Final = frozenset({"killed", "survived", "total"})
+
+
+def unresolved_counts(stats: dict) -> dict[str, int]:
+    """Every non-zero count that is neither a kill nor a survivor means a mutant did not
+    reach the tests. Reading it as "anything else" rather than as a list of known statuses
+    keeps a status this reporter has never met from passing as a clean sweep."""
+    return {k: v for k, v in sorted(stats.items()) if k not in RESOLVED_KEYS and isinstance(v, int) and v > 0}
+
+
+def clean_sweep_is_provable(stats: dict | None) -> bool:
+    """`mutmut results` omits killed mutants, so its silence is equally consistent with a
+    perfect run and with a run that never started. Only the stats file can tell them apart,
+    and only when it agrees that nothing survived and every mutant reached the tests."""
+    if not stats or stats.get("killed", 0) <= 0 or stats.get("survived", 0) != 0:
+        return False
+    return not unresolved_counts(stats)
+
+
+def no_survivors_verdict(results: MutmutResults, stats: dict | None) -> str:
+    if clean_sweep_is_provable(stats):
+        return "**No surviving mutants, and the run killed some, so the test suite caught every mutation.**"
+    if stats and stats.get("survived", 0) > 0:
+        return (
+            f"**mutmut-cicd-stats.json counts {stats['survived']} surviving mutant(s) that "
+            "`mutmut results` did not list, so the two disagree and neither can be trusted. "
+            "This is not a passing score.**"
+        )
+    if stats and unresolved_counts(stats):
+        unresolved = ", ".join(f"{v} {k.replace('_', ' ')}" for k, v in unresolved_counts(stats).items())
+        return (
+            f"**No survivors, but {unresolved}, so those mutants never reached the tests "
+            "and the suite was not shown to catch them. This is not a passing score.**"
+        )
+    if stats:
+        return "**Not one mutant was killed. This is not a passing score.**"
+    return (
+        f"**mutmut-cicd-stats.json is missing and `mutmut results` printed {results.reported} "
+        "verdict(s), none of them a survivor. Since that command never lists killed mutants, a "
+        "clean sweep and a run that mutated nothing look identical from here. This is not a "
+        "passing score.**"
+    )
+
+
+def render(config: dict, results: MutmutResults, stats: dict | None) -> str:
+    survivors = list(results.survivors)
     by_function: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list)
     for survivor in survivors:
         module_path, function_name, mutant_num = parse_mutant_name(survivor)
@@ -235,17 +289,8 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str:
     out.append("## Summary")
     out.append("")
     if stats:
-        total = stats.get("total", 0) or sum(
-            stats.get(k, 0)
-            for k in (
-                "killed",
-                "survived",
-                "no_tests",
-                "skipped",
-                "suspicious",
-                "timeout",
-                "segfault",
-            )
+        total = stats.get("total", 0) or (
+            stats.get("killed", 0) + stats.get("survived", 0) + sum(unresolved_counts(stats).values())
         )
         killed = stats.get("killed", 0)
         survived = stats.get("survived", 0)
@@ -254,17 +299,15 @@ def render(config: dict, survivors: list[str], stats: dict | None) -> str:
         out.append(f"- Killed: **{killed}**")
         out.append(f"- Survived: **{survived}**")
         out.append(f"- Mutation score: **{score:.1f}%**")
-        for k in ("no_tests", "skipped", "suspicious", "timeout", "segfault"):
-            v = stats.get(k, 0)
-            if v:
-                out.append(f"- {k.replace('_', ' ').title()}: {v}")
+        for k, v in unresolved_counts(stats).items():
+            out.append(f"- {k.replace('_', ' ').title()}: {v}")
     else:
         out.append(f"- Survivors found: **{len(survivors)}**")
         out.append("- (mutmut-cicd-stats.json not available — full counts unavailable)")
     out.append("")
 
     if not survivors:
-        out.append("**No surviving mutants — the test suite caught every mutation.**")
+        out.append(no_survivors_verdict(results, stats))
         out.append("")
         return "\n".join(out)
 
@@ -407,15 +450,22 @@ def main() -> int:
         except json.JSONDecodeError as exc:
             print(f"warning: could not parse {stats_file}: {exc}", file=sys.stderr)
 
-    survivors = get_survivors()
-    report = render(config, survivors, stats)
+    results = get_survivors()
+    report = render(config, results, stats)
 
     out_path = ROOT / "mutation-report.md"
     out_path.write_text(report)
     print(
-        f"Wrote {out_path} ({len(survivors)} survivor"
-        f"{'s' if len(survivors) != 1 else ''}, {len(report)} chars)"
+        f"Wrote {out_path} ({len(results.survivors)} survivor"
+        f"{'s' if len(results.survivors) != 1 else ''}, {len(report)} chars)"
     )
+    if not results.survivors and not clean_sweep_is_provable(stats):
+        print(
+            "error: nothing was shown to have been killed, so the report cannot say "
+            "anything about the suite",
+            file=sys.stderr,
+        )
+        return 1
     return 0
 
 

terraform/provider/.goreleaser.yml

@@ -72,6 +72,7 @@ signs:
       - "--detach-sign"
       - "${artifact}"
 release:
+  prerelease: auto
   extra_files:
     - glob: 'terraform-registry-manifest.json'
       name_template: '{{ .ProjectName }}_{{ .Version }}_manifest.json'

terraform/provider/CHANGELOG.md

@@ -2,11 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+Up to `0.4.0` the provider had its own version line, cut from the headings in
+this file. It now ships at the **LiteLLM version**, on every LiteLLM release
+channel, built from the same commit as the proxy (see `RELEASING.md`). The
+headings below no longer drive a release; they record what changed and which
+LiteLLM line first carried it. A change that breaks existing configurations
+or state must be called out loudly here, because the version number can no
+longer signal it.
 
 ## [Unreleased]
 
+### Changed
+
+- **Versioning**: the provider is now published at the LiteLLM version, from the same commit as the proxy, on every LiteLLM release (dev, rc, stable). The `0.x` line ends at `0.4.0`; a `~> 0.4` constraint will not receive further releases, so re-pin to the LiteLLM version your proxy runs (for example `~> 1.99.0`). Existing `0.x` versions remain in the registry and keep verifying
+
 ## [0.4.0] - 2026-08-06
 
 ### Fixed

terraform/provider/README.md

@@ -6,6 +6,18 @@ This Terraform provider allows you to manage LiteLLM resources through Infrastru
 
 This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
 
+## Versioning
+
+The provider version **is the LiteLLM version**. Every LiteLLM release (dev, rc and stable) publishes the provider at the same version as the proxy, built from the same commit, so `1.99.0` of the provider is the one that shipped with `1.99.0` of the proxy and was audited against that proxy's API. Pin the provider to the line your proxy runs:
+
+```hcl
+version = "~> 1.99.0"
+```
+
+Pre-release versions (`1.99.0-rc.1`, `1.99.0-dev.1`) are published too; Terraform only selects one when it is pinned exactly.
+
+Versions `0.1.0` through `0.4.0` predate this scheme and sit on their own line. They stay in the registry, but **a `~> 0.4` constraint will never pick up another release**: re-pin to the LiteLLM version to keep receiving updates.
+
 ## Features
 
 - Manage LiteLLM model configurations
@@ -32,7 +44,7 @@ terraform {
   required_providers {
     litellm = {
       source  = "BerriAI/litellm"
-      version = "~> 0.1.1" #HERE UPDATE VERSION ACCORDINGLY
+      version = "~> 1.99.0" # the LiteLLM version your proxy runs
     }
   }
 }
@@ -218,6 +230,6 @@ This project is licensed under the Apache License 2.0 - see the [LICENSE](LICENS
 
 - Always use environment variables or secure secret management solutions to handle sensitive information like API keys and AWS credentials.
 - Refer to the comprehensive documentation in the `docs/` directory for detailed usage examples and configuration options.
-- Make sure to keep your provider version updated for the latest features and bug fixes.
+- Keep the provider version in step with the LiteLLM version your proxy runs; see [Versioning](#versioning).
 - The provider now supports AWS cross-account access with `aws_session_name` and `aws_role_name` parameters in the model resource.
 - All example configurations have been consolidated into the documentation for better organization and maintenance.

terraform/provider/RELEASING.md

@@ -4,7 +4,16 @@ This document describes the release process for the LiteLLM Terraform Provider.
 
 ## Overview
 
-Releases are automated via GitHub Actions when a version tag is pushed. The workflow builds the provider for multiple platforms, signs the artifacts with GPG, and publishes them to GitHub Releases.
+The provider is released **in lockstep with LiteLLM**: every LiteLLM release (dev, rc and stable) publishes the provider at the LiteLLM version, built from the same commit as the proxy. There is no separate provider release to cut.
+
+The flow, end to end:
+
+1. `BerriAI/project-releaser`'s release pipeline resolves the commit to release (`main` HEAD for dev; `main` HEAD or an operator-supplied SHA for rc/stable) and passes the release approval gate
+2. Its componentized terraform job rsyncs `terraform/provider/` from that commit into `BerriAI/terraform-provider-litellm`, commits, and pushes the tag `v<litellm version>` (for example `v1.99.0`, `v1.99.0-rc.1`, `v1.99.0-dev.1`), alongside the `terraform-aws-litellm` / `terraform-google-litellm` module mirrors which get the same tag
+3. The tag push triggers the mirror's own `Release` workflow (goreleaser): multi-platform build, GPG-signed checksums, GitHub release. It runs unattended; project-releaser does not wait for it
+4. The public Terraform Registry ingests the GitHub release as provider version `<litellm version>`
+
+`terraform/provider/` only exists from LiteLLM ~1.95, so a stable patch cut from an older line skips the provider and publishes only the modules.
 
 ## Prerequisites
 
@@ -68,113 +77,26 @@ Before publishing to the Terraform Registry:
 
 **Note**: The public key fingerprint must match the key used to sign the provider releases.
 
-## Release Steps
+## What a change needs
 
-### 1. Prepare the Release
+1. **Land it in `BerriAI/litellm`.** Open a PR against `litellm_internal_staging` with the source change and a `CHANGELOG.md` entry under `[Unreleased]`. CI runs `gofmt`, `go vet`, build, tests and the endpoint-drift audit. A change that breaks existing configurations or state must say so in the changelog: the version number cannot signal it any more
+2. **Wait for the next LiteLLM release.** The nightly dev release carries it within a day; it reaches a stable version on the next stable cut
+3. **Verify** (optional): the version appears at https://registry.terraform.io/providers/BerriAI/litellm and https://github.com/BerriAI/terraform-provider-litellm/releases. If the tag is on the mirror but there is no release, the goreleaser run failed: https://github.com/BerriAI/terraform-provider-litellm/actions
 
-Before creating a release:
+Locally, before opening the PR:
 
-1. **Update CHANGELOG.md**
-   - Move items from `[Unreleased]` section to a new version section
-   - Follow [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format
-   - Use [Semantic Versioning](https://semver.org/spec/v2.0.0.html) for version numbers
-   - Include all notable changes since the last release
+```bash
+make test
+make build
+```
 
-   Example:
-   ```markdown
-   ## [0.1.2] - 2026-02-20
+## Out-of-band publish or recovery
 
-   ### Added
-   - New feature description
+Dispatch `Build and Publish Componentized Images + Chart` in `BerriAI/project-releaser` by hand with only `publish_terraform` enabled and the `git_ref` / `tag` of the release to (re)publish. The run waits on project-releaser's release approval, then mirrors and tags exactly as the pipeline does.
 
-   ### Fixed
-   - Bug fix description
+The mirror is push-only: do not commit or tag `BerriAI/terraform-provider-litellm` directly. The publish refuses to overwrite an existing tag; a version that failed in goreleaser is recovered by re-running the mirror's `Release` workflow for that tag, not by re-tagging.
 
-   ### Changed
-   - Changed behavior description
-   ```
-
-2. **Verify tests pass**
-   ```bash
-   make test
-   ```
-
-3. **Verify the build works locally**
-   ```bash
-   make build
-   ```
-
-4. **Land the changes in BerriAI/litellm**
-
-   Open a PR to `BerriAI/litellm` updating `terraform/provider/CHANGELOG.md` (and any source changes) and merge it
-
-### 2. Mirror and Tag via project-releaser
-
-The provider source lives at `terraform/provider/` in `BerriAI/litellm`; `BerriAI/terraform-provider-litellm` is a thin release mirror. Do not commit or tag the mirror directly
-
-Normally there is nothing to do here. `BerriAI/project-releaser`'s release pipeline runs the same check on every release except `adhoc`, nightly included: it reads the topmost released heading in `terraform/provider/CHANGELOG.md`, probes the mirror for `v<version>`, and dispatches `Publish Terraform provider` only when the changelog has moved ahead of what the mirror carries. Cutting the version heading in step 1 is therefore what releases the provider, and the next release picks it up, so the wait is a day rather than a week
-
-Dispatch by hand only for an out-of-band release, or to recover a run that failed:
-
-1. Go to `BerriAI/project-releaser` > **Actions** > `Publish Terraform provider`
-2. Click **Run workflow**:
-   - `git_ref`: full 40-char commit SHA from `BerriAI/litellm` to release from
-   - `provider_version`: the new version without the `v` prefix (e.g. `0.3.0`)
-   - `dry_run`: optional; validates without pushing
-
-Automatic or manual, the run waits on the `production-release` approval in `project-releaser`, then rsyncs `terraform/provider/` into the mirror repo, commits, and pushes tag `v<provider_version>`. That approval is the only one in the flow. The tag push triggers the mirror's `Release` workflow (goreleaser), which runs unattended
-
-**Important**:
-- Tags must follow the format: `v<MAJOR>.<MINOR>.<PATCH>` (e.g., `v0.1.2`, `v1.0.0`)
-- The workflow refuses to overwrite an existing tag; publish a new version instead
-
-### 3. Monitor the Release Workflow
-
-1. Go to: https://github.com/BerriAI/terraform-provider-litellm/actions
-2. Find the "Release" workflow run for your tag
-3. Monitor the progress and check for any errors
-
-The workflow will:
-- Check out the code
-- Set up Go
-- Import the GPG key
-- Run `go mod tidy`
-- Build binaries for multiple platforms (Linux, macOS, Windows, FreeBSD)
-- Create archives and checksums
-- Sign the checksums with GPG
-- Create a GitHub release
-- Upload all artifacts
-
-### 4. Verify the Release
-
-After the workflow completes successfully:
-
-1. **Check the GitHub Release**
-   - Go to: https://github.com/BerriAI/terraform-provider-litellm/releases
-   - Verify the release was created with the correct version
-   - Confirm all artifacts are present:
-     - Binary archives for each platform
-     - SHA256SUMS file
-     - SHA256SUMS.sig (GPG signature)
-     - terraform-registry-manifest.json
-
-2. **Verify the signature** (optional)
-   ```bash
-   # Download the checksums and signature
-   wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS
-   wget https://github.com/BerriAI/terraform-provider-litellm/releases/download/v0.1.2/terraform-provider-litellm_0.1.2_SHA256SUMS.sig
-
-   # Verify the signature
-   gpg --verify terraform-provider-litellm_0.1.2_SHA256SUMS.sig terraform-provider-litellm_0.1.2_SHA256SUMS
-   ```
-
-### 5. Publish to Terraform Registry (Optional)
-
-If this provider is published to the Terraform Registry:
-
-1. The registry should automatically detect the new release via the GitHub webhook
-2. If not, you may need to manually trigger a sync on the Terraform Registry dashboard
-3. Verify the new version appears at: https://registry.terraform.io/providers/BerriAI/litellm/latest
+The mirror's `.github/` directory (the `Release` workflow) is the one thing the rsync preserves, so a change to the goreleaser *workflow* is a direct PR on the mirror; a change to `.goreleaser.yml` itself lands here like any other source change.
 
 ## Troubleshooting
 
@@ -207,21 +129,15 @@ If this provider is published to the Terraform Registry:
 
 ### Tag Already Exists
 
-**Error**: The publish workflow refuses to push because the tag already exists on the mirror
+**Error**: The publish job refuses to push because the tag already exists on the mirror
 
-**Solution**: Tags are immutable by design. Re-run the workflow with a new patch version instead of deleting or moving an existing tag
+**Solution**: Tags are immutable by design and the version is the LiteLLM version, so this means the provider was already mirrored for this release. If the registry is missing the version, re-run the mirror's `Release` workflow for the existing tag rather than re-tagging
 
 ## Version Numbering
 
-This project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html):
+The provider version is the LiteLLM version, verbatim: `X.Y.Z` for a stable release, `X.Y.Z-rc.N` for a release candidate and `X.Y.Z-dev.N` for a nightly. It says which proxy the provider shipped with and was audited against; it does not follow SemVer's break-signalling, so breaking changes are announced in `CHANGELOG.md` and the registry docs instead.
 
-- **MAJOR** version (1.0.0): Incompatible API changes
-- **MINOR** version (0.1.0): New functionality in a backward-compatible manner
-- **PATCH** version (0.0.1): Backward-compatible bug fixes
-
-For pre-1.0 releases:
-- Breaking changes may occur in minor versions
-- Patch versions should only contain bug fixes
+Versions `0.1.0` to `0.4.0` predate this and remain in the registry on their own line. A `~> 0.4` constraint never receives another release.
 
 ## Security Considerations
 
@@ -237,5 +153,4 @@ For pre-1.0 releases:
 - [Terraform Provider Publishing](https://www.terraform.io/docs/registry/providers/publishing.html)
 - [HashiCorp GPG Signing Requirements](https://www.terraform.io/docs/registry/providers/publishing.html#signing-releases)
 - [GitHub Actions Secrets](https://docs.github.com/en/actions/security-guides/encrypted-secrets)
-- [Semantic Versioning](https://semver.org/)
 - [Keep a Changelog](https://keepachangelog.com/)

test-quality-budget.json

@@ -1,18 +1,18 @@
 {
   "TQ001": {
-    "limit": 750
+    "limit": 744
   },
   "TQ002": {
     "limit": 742
   },
   "TQ003": {
-    "limit": 1078
+    "limit": 1068
   },
   "TQ004": {
-    "limit": 768
+    "limit": 469
   },
   "TQ005": {
-    "limit": 2832
+    "limit": 2459
   },
   "TQ006": {
     "limit": 34

tests/audio_tests/test_audio_speech.py

@@ -12,7 +12,6 @@ from litellm._uuid import uuid
 from dotenv import load_dotenv
 
 load_dotenv()
-import os
 
 sys.path.insert(
     0, os.path.abspath("../..")
@@ -452,7 +451,7 @@ async def test_azure_ava_tts_with_custom_voice():
     Test that when using a custom Azure voice (en-US-AndrewNeural),
     the SSML request body contains the selected voice.
     """
-    from unittest.mock import AsyncMock, MagicMock, patch
+    from unittest.mock import AsyncMock, patch
     import httpx
 
     # Mock response
@@ -497,7 +496,7 @@ async def test_azure_ava_tts_fable_voice_mapping():
     Test that when using OpenAI voice 'fable',
     it gets mapped to Azure voice 'en-GB-RyanNeural' in the SSML.
     """
-    from unittest.mock import AsyncMock, MagicMock, patch
+    from unittest.mock import AsyncMock, patch
     import httpx
 
     # Mock response
@@ -544,7 +543,7 @@ async def test_aws_polly_tts_with_native_voice():
     Verifies the request is formatted correctly for the Polly API.
     """
     import json
-    from unittest.mock import MagicMock, patch
+    from unittest.mock import patch
     import httpx
 
     # Mock response - Polly returns audio bytes directly
@@ -592,7 +591,7 @@ async def test_aws_polly_tts_with_openai_voice_mapping():
     Verifies that OpenAI voices are correctly mapped to Polly voices.
     """
     import json
-    from unittest.mock import MagicMock, patch
+    from unittest.mock import patch
     import httpx
 
     mock_response_content = b"fake_audio_data"
@@ -634,7 +633,7 @@ async def test_aws_polly_tts_with_ssml():
     Verifies that SSML is detected and TextType is set correctly.
     """
     import json
-    from unittest.mock import MagicMock, patch
+    from unittest.mock import patch
     import httpx
 
     mock_response_content = b"fake_audio_data"

tests/audio_tests/test_whisper.py

@@ -44,7 +44,6 @@ load_dotenv()
 sys.path.insert(
     0, os.path.abspath("../")
 )  # Adds the parent directory to the system path
-import litellm
 from litellm import Router
 
 
@@ -146,7 +145,6 @@ async def test_whisper_log_pre_call():
     from litellm.litellm_core_utils.litellm_logging import Logging
     from datetime import datetime
     from unittest.mock import patch, MagicMock
-    from litellm.integrations.custom_logger import CustomLogger
 
     custom_logger = CustomLogger()
 

tests/e2e/CLAUDE.md

@@ -77,13 +77,26 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
 
 The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers
 
-A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket
+A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. `fixture_bundle.py` owns the format. Record serves the proxy the same filtered stored response replay will serve later, so the two modes are byte-identical from the proxy's side of the socket
+
+Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as `<secret>`, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `<sha256>` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length
 
 Replay matches calls per test by canonical key: `fixture_canonical.py` canonicalizes the recorded request (volatile headers and credential fields out, unique markers, generated ids, uuids, and timestamps replaced with fixed placeholders, object keys sorted) and the key is the method, edge path, and a content hash, so identity survives re-records and machine changes while any real content drift comes back as an HTTP 599 naming the computed key, the closest recorded key with its file, and a content diff, and never falls through to a live call. Matching is order-independent across distinct keys (concurrent calls may interleave) and FIFO within one key (a retry loop replays its responses in recorded order); a passed test must also consume its whole recording, or teardown fails it naming a leftover key. Either way the fix is always to re-record with `E2E_FIXTURE_MODE=record`. Every rewrite rule lives in `fixture_canonical.py`, so a new volatile header, credential field name, or generated-id shape is one edit there. Record starts fresh every time: it wipes the previous bundle (refusing to wipe a directory that is not a bundle) and never reads it. A replay bundle whose manifest is older than seven days hard-fails at collection time naming the bundle's age, so replay can never certify against fixtures that have drifted more than a week from the live providers
 
 A replayed response carries the recorded provider response id, and `LiteLLM_SpendLogs.request_id` (the table's primary key) is that id, so a replay against a database that still holds the record run's rows silently dedupes its spend inserts and any spend assertion goes red with zero matching rows and nothing in the proxy log. Run both modes with `E2E_RESET_SPEND_LOGS=1` (plus `DATABASE_URL` in the runner env) so each session truncates the table after itself, or replay against a fresh database, which is the CI shape
 
-Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), multipart uploads have per-run random boundaries (the digest changes every run, so they always miss), and deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base)
+The same id reuse reaches the managed-object tables. A replayed `/v1/files` or `/v1/batches` response carries the recorded provider object id, and `LiteLLM_ManagedObjectTable.model_object_id` is unique, so a unified batch create replayed against a database that still holds the record run's row fails on a Prisma unique-constraint violation, which surfaces as a 500, makes the router retry, and exhausts the recording. Replay the batches suite against a fresh database, or truncate `LiteLLM_ManagedObjectTable` and `LiteLLM_ManagedFileTable` before the run
+
+Edge-wired today: `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` (the reference), `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the Anthropic deployments in `llm_translation/test_messages_e2e.py` except the streaming test, and the OpenAI batch deployment behind `batches/` (`capabilities.openai_batch_params`). The mount base is not the same for both providers: OpenAI deployments register `f"{base}/v1"`, Anthropic deployments register `base` on its own, because litellm's Anthropic handler appends `/v1/messages` to `api_base` itself where the OpenAI handler appends only `/chat/completions`. Recording one suite locally is two runs against a proxy you already have up:
+
+```bash
+E2E_FIXTURE_MODE=record E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 uv run pytest tests/e2e/llm_translation/test_chat_completions_contract_e2e.py
+E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures E2E_RESET_SPEND_LOGS=1 uv run pytest tests/e2e/llm_translation/test_chat_completions_contract_e2e.py
+```
+
+Point the proxy at bogus provider credentials for the replay run and it still has to pass: that is the whole proof that nothing left the process. Bundles are never committed. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and hard-fails after seven days, and publishing one for CI is LIT-5748
+
+Current limits: streaming chunk fidelity is LIT-5742 (a streamed response records as one buffered body), CI wiring is LIT-5748, Bedrock cannot be mounted (SigV4 signs the Host header, so a rewritten api_base fails signature verification), deployments baked into the proxy's config file cannot be edge-wired (only `/model/new` registrations can carry the edge api_base), and a file upload routed by `custom_llm_provider` through the proxy's `files_settings` block never passes a deployment at all, so the batches `model_param` and `provider_fallback` scenarios keep uploading live in every mode
 
 ## Typing
 

tests/e2e/CONTRIBUTING.md

@@ -57,13 +57,15 @@ Some suites need extra services the bare proxy does not start. The `logging/` OT
 Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop
 
 ```bash
-E2E_FIXTURE_MODE=record uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v
-E2E_FIXTURE_MODE=replay uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v
+E2E_FIXTURE_MODE=record E2E_FIXTURE_DIR=/tmp/e2e-fixtures uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v
+E2E_FIXTURE_MODE=replay E2E_FIXTURE_DIR=/tmp/e2e-fixtures uv run pytest tests/e2e/quota_management/spend_tracking/test_provider_edge_spend_e2e.py -v
 ```
 
+Bundles stay local. `tests/e2e/.fixtures` is gitignored because a bundle holds verbatim provider response bodies and expires seven days after it was recorded, so record the suite you want before you replay it and never commit the result; publishing bundles for CI is LIT-5748
+
 One sharp edge: a replayed response reuses the recorded provider response id, and that id is the primary key of `LiteLLM_SpendLogs`, so replaying against a database that still holds the record run's rows silently dedupes the spend writes and a spend assertion fails with zero rows. Run both commands above with `E2E_RESET_SPEND_LOGS=1` (and `DATABASE_URL` set in the pytest env) so each session truncates the spend log table after itself, or point replay at a fresh database
 
-Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock, multipart)
+Replay answers any provider call that drifted from the recording with an HTTP 599 whose body names the computed and closest recorded keys, so the test fails loudly instead of silently going live, and a bundle older than seven days fails at collection time naming its age; either way the fix is to re-record. Only tests that register edge-wired deployments participate: everything else hits its provider live in every mode, so record exactly the suite you replay. If the proxy runs in a container, set `E2E_PROVIDER_EDGE_ADVERTISE_HOST` (e.g. `host.docker.internal`) so the api_base the proxy stores can reach the edge on the pytest host, and `E2E_PROVIDER_EDGE_BIND_HOST=0.0.0.0` so the edge accepts it. The suites wired to the edge today are `quota_management/spend_tracking/test_provider_edge_spend_e2e.py`, `llm_translation/test_chat_completions_contract_e2e.py`, the OpenAI registrations in `llm_translation/test_embeddings_endpoint_e2e.py`, the non-streaming Anthropic tests in `llm_translation/test_messages_e2e.py`, and the OpenAI batch deployment behind `batches/`. See `CLAUDE.md` in this directory for the bundle format, the edge design, and the current limits (streaming, Bedrock)
 
 Tests marked `@pytest.mark.e2e` hard-fail when no proxy answers `/health/liveliness`, so a run that goes red with `No live proxy` at setup means the proxy isn't up; they never skip for a missing proxy, so an absent proxy can't be mistaken for a pass
 

tests/e2e/batches/COVERAGE.md

@@ -1,9 +1,11 @@
 # Batches Test Coverage Matrix
 
 Live e2e coverage of the Batches API over a real proxy, real provider keys, and
-real cost. Synchronous tier only: a batch's completion window is 24h, so these
-tests never wait for `completed`. They assert the proxy accepts, routes, retrieves,
-cancels, and lists a batch; everything created is deleted on teardown.
+real cost. Mostly synchronous tier: a batch's completion window is 24h, so the
+lifecycle matrix never waits for `completed`. It asserts the proxy accepts, routes,
+retrieves, cancels, and lists a batch; everything created is deleted on teardown.
+The exception is `TestBatchTerminalState`, which covers the completed state and
+cost write-back via a cross-run marker baton (design below).
 
 ## Provider x operation
 
@@ -12,19 +14,26 @@ row per supported (provider, scenario) pair, so there are no skipped cells in th
 parametrized run. The batches suite never skips: missing provider creds or upstream
 failures are hard test failures (see `tests/e2e/CLAUDE.md`).
 
-| Provider  | create | retrieve | cancel | list | file backing |
-|-----------|--------|----------|--------|------|--------------|
-| OpenAI    | yes | yes | yes | yes | OpenAI Files |
-| Azure     | yes | yes | yes | yes | Azure Files |
-| Vertex AI | yes | yes | yes | yes | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) |
-| Bedrock   | yes (unified only) | yes | no (limited upstream) | no | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
+| Provider  | create | retrieve | cancel | list | content download | file backing |
+|-----------|--------|----------|--------|------|------------------|--------------|
+| OpenAI    | yes | yes | yes | yes | yes (lifecycle + terminal output) | OpenAI Files |
+| Azure     | yes | yes | yes | yes | yes (byte-verbatim) | Azure Files |
+| Vertex AI | yes | yes | yes | yes | yes (provider-transformed) | GCS (`gcs_bucket_name` / `GCS_BUCKET_NAME` on model) |
+| Bedrock   | yes (unified only) | yes | no (limited upstream) | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
 
 Bedrock cancel is unreliable upstream and list is unsupported, so both are gated off
-(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix.
+(`can_cancel=False`, `can_list=False`) when that provider is enabled in the matrix;
+flipping those gates is tracked in LIT-4774 and deliberately not part of this suite.
 Bedrock file upload requires a model on the request (`encoded` / `unified` scenarios only);
 `model_param` and `provider_fallback` are omitted because `POST /bedrock/v1/files` has no
 model-less passthrough path.
 
+`GET /v1/files/{id}/content` is exercised for the unified upload path per backend in
+`test_unified_file_content_downloads`. Azure stores the JSONL verbatim, so its download
+is asserted byte-equal to the upload. Vertex (GCS) and Bedrock (S3) transform lines at
+upload time, so those assert a 200 with non-empty parseable JSON lines instead. Gemini
+(non-Vertex) raises `NotImplementedError` for file content and has no cell here.
+
 ## Routing scenarios (per `litellm/proxy/batches_endpoints/endpoints.py`)
 
 Each create-capable provider runs all four. The test asserts the returned file id
@@ -71,11 +80,59 @@ File delete asserts `object=="file"` and `deleted==True`.
 | `batch_client.py` | typed file upload/download + batch create/retrieve/cancel/list/delete over the shared ProxyClient; runtime batch model registration via /model/new; denial helpers |
 | `capabilities.py` | the provider x scenario matrix + per-provider /model/new params + id-shape classifiers + per-provider raw-id assertion |
 | `conftest.py` | session-scoped batch deployment registration and teardown |
-| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial |
+| `test_batches_e2e.py` | parametrized lifecycle with per-endpoint output assertions, file upload/delete outputs, key-model-access denial, per-backend content download, failure paths, second-hop routing, terminal state + cost |
+
+## Failure paths
+
+`TestBatchFailurePaths` pins the customer-facing error contracts. A malformed input
+file is a 400 at upload naming the bad content. A JSONL line whose url contradicts
+the batch endpoint passes create (providers validate asynchronously) and drives the
+batch to `failed` with structured `errors.data` (code/line/message), a null
+`output_file_id`, and a $0 spend row keyed `{batch_id}_batch_cost` (LIT-4852: a
+failed batch books $0 instead of crashing cost tracking). Cancelling that failed
+batch is a 409 naming the terminal status. A file id encoded for one deployment wins
+over a conflicting `model` param on create: the batch routes and re-encodes by the
+file's embedded model (foreign-id precedence).
+
+## Second hop (two chained gateways)
+
+`TestBatchSecondHop` registers a `litellm_proxy/<inner model>` deployment pointing at
+the proxy's own base URL with a freshly minted virtual key, so unified upload and
+create traverse gateway -> gateway -> OpenAI (LIT-5347, PR #36240). The pin:
+`target_model_names` is rewritten to the inner deployment on the second hop and the
+nested managed ids round-trip retrieve. This self-chaining only needs the proxy to
+reach its own `PROXY_BASE_URL`, which holds both locally and on the e2e stage.
+
+## Terminal state + cost write-back (cross-run marker baton)
+
+The 24h completion window rules out submit-and-wait inside one run, so
+`TestBatchTerminalState` amortizes across runs. Each run submits a 1-line marker
+batch (stable metadata key/value plus a per-run field) and deliberately never
+cancels or deletes it or its input file: the marker is the baton the next run picks
+up (OpenAI files expire on their own after ~30 days). Polling is list-only, up to 5
+minutes, because retrieving a non-terminal batch books a $0 spend row whose
+request_id then blocks the later real-cost row (`skip_duplicates`); the single
+retrieve happens only once a completed marker exists. The assertion target is the
+newest completed marker from ANY run: run-scoped deployment names mean the list
+re-encodes prior-run batches under new encoded ids, so their spend keys are fresh
+and a prior-run marker is billable by this run. On the 6h stage cadence the full
+assertions are therefore deterministic from run 2 onward. On a cold start (no
+completed marker within the poll budget) the test passes on the submission
+assertions alone: a documented vacuous pass, not a skip. Markers aged past the 24h
+window (25h-73h band, within the newest 100-item list page) must be terminal.
+
+The cost assertion is the LIT-5730 headline: retrieving a completed model-encoded
+batch must write a positive spend row with call_type `aretrieve_batch` and token
+usage. Before the fix in `litellm/batches/batch_utils.py`, the retrieve endpoint
+re-encoded the response's `output_file_id` in place before the queued logging
+worker ran, the worker sent that encoded id to OpenAI, got a 404, and the spend row
+never landed.
 
 ## Out of scope (intentionally)
 
-Driving a batch to `completed`, cost tracking on completion, and the DB write-back
-are not covered here; the 24h window makes them unfit for a synchronous gate. That
-logic belongs in a DI-stubbed proxy integration test under `tests/test_litellm/proxy/`
-where the provider client is injected to return `completed` deterministically.
+Unified (managed) batch cost is owned by the hourly `CheckBatchCost` poller, and a
+terminal DB status short-circuits retrieve for those ids, so the terminal-state cell
+uses the encoded path; poller timing does not fit an e2e gate and belongs in a
+DI-stubbed proxy integration test under `tests/test_litellm/proxy/`. Bedrock
+cancel/list stay gated pending LIT-4774. Gemini (non-Vertex) file content raises
+`NotImplementedError` upstream and is not a coverage cell.

tests/e2e/batches/batch_client.py

@@ -51,6 +51,17 @@ class FileList(BaseModel):
     has_more: bool | None = None
 
 
+class BatchErrorItem(BaseModel):
+    code: str | None = None
+    line: int | None = None
+    message: str | None = None
+
+
+class BatchErrorList(BaseModel):
+    object: str | None = None
+    data: list[BatchErrorItem] = []
+
+
 class BatchObject(BaseModel):
     id: str
     object: str | None = None
@@ -58,6 +69,9 @@ class BatchObject(BaseModel):
     endpoint: str | None = None
     input_file_id: str | None = None
     output_file_id: str | None = None
+    error_file_id: str | None = None
+    errors: BatchErrorList | None = None
+    metadata: dict[str, str] | None = None
     completion_window: str | None = None
     created_at: int | None = None
     model: str | None = None
@@ -79,12 +93,18 @@ class BatchCreateBody(BaseModel):
     endpoint: str = "/v1/chat/completions"
     completion_window: str = "24h"
     model: str | None = None
+    metadata: dict[str, str] | None = None
 
 
 class ModelQuery(BaseModel):
     model: str | None = None
 
 
+class BatchListQuery(BaseModel):
+    model: str | None = None
+    limit: int | None = None
+
+
 def is_model_access_denied(resp: StreamingResponse) -> bool:
     """True if the proxy rejected the call because the key may not access the model."""
     return resp.status_code == 403 and "key_model_access_denied" in resp.body
@@ -175,12 +195,17 @@ class BatchClient:
         )
 
     def list_batches(
-        self, *, key: str, provider: str | None = None
+        self,
+        *,
+        key: str,
+        provider: str | None = None,
+        model: str | None = None,
+        limit: int | None = None,
     ) -> Result[BatchList]:
         return self.proxy.transport.get(
             _batches_path(provider),
             headers=self.proxy.transport.bearer(key),
-            params=NoBody(),
+            params=BatchListQuery(model=model, limit=limit),
             response_type=BatchList,
         )
 

tests/e2e/batches/capabilities.py

@@ -5,9 +5,9 @@ from __future__ import annotations
 import base64
 import os
 from dataclasses import dataclass
-from typing import Literal
+from typing import Final, Literal
 
-from e2e_config import unique_marker
+from e2e_config import provider_edge_base, unique_marker
 from models import LiteLLMParamsBody
 
 _BATCH_RUN = unique_marker()
@@ -17,6 +17,21 @@ def batch_model_name(base: str) -> str:
     return f"{base}-{_BATCH_RUN}"
 
 
+OPENAI_BATCH_BACKEND: Final = "gpt-4o-mini"
+
+
+def openai_batch_params() -> LiteLLMParamsBody:
+    """The OpenAI batch deployment, wired through the record/replay edge when a fixture
+    mode is active and straight at OpenAI otherwise (LIT-5974). Azure, Vertex, and
+    Bedrock stay live: none of them has an edge mount."""
+    base = provider_edge_base("openai")
+    return LiteLLMParamsBody(
+        model=f"openai/{OPENAI_BATCH_BACKEND}",
+        api_key="os.environ/OPENAI_API_KEY",
+        api_base=None if base is None else f"{base}/v1",
+    )
+
+
 def _env_ref(*names: str) -> str:
     for name in names:
         value = os.environ.get(name)
@@ -47,10 +62,7 @@ class Provider:
     def litellm_params(self) -> LiteLLMParamsBody:
         match self.name:
             case "openai":
-                return LiteLLMParamsBody(
-                    model="openai/gpt-4o-mini",
-                    api_key="os.environ/OPENAI_API_KEY",
-                )
+                return openai_batch_params()
             case "azure":
                 return LiteLLMParamsBody(
                     model="azure/gpt-5.4-mini-batch",
@@ -107,7 +119,11 @@ class Capability:
 
 PROVIDERS: tuple[Provider, ...] = (
     Provider(
-        "openai", batch_model_name("openai-batch"), "gpt-4o-mini", can_cancel=True, can_list=True
+        "openai",
+        batch_model_name("openai-batch"),
+        OPENAI_BATCH_BACKEND,
+        can_cancel=True,
+        can_list=True,
     ),
     Provider(
         "azure",
@@ -210,6 +226,16 @@ def is_model_encoded_id(id_str: str) -> bool:
     return False
 
 
+def decoded_model_from_id(id_str: str) -> str | None:
+    """Deployment name embedded in a model-encoded file/batch id, or None."""
+    for prefix in ("file-", "batch_"):
+        if id_str.startswith(prefix):
+            decoded = _b64_decode(id_str[len(prefix) :])
+            if decoded.startswith("litellm:") and ";model," in decoded:
+                return decoded.split(";model,", 1)[1].split(";")[0]
+    return None
+
+
 def matches_id_shape(shape: IdShape, id_str: str) -> bool:
     if shape == "managed":
         return is_managed_id(id_str)

tests/e2e/batches/test_batches_e2e.py

@@ -1,11 +1,12 @@
 """Live e2e for the Batches API across every provider LiteLLM supports.
 
-Synchronous tier only: a batch's completion window is 24h, so these never wait for
-"completed". Each case uploads a tiny JSONL, creates the batch through one of the
-four routing scenarios, asserts it was accepted (non-terminal status) and routed to
-the right provider, then retrieves / cancels / lists where the provider supports it.
-Everything created is deleted on teardown. Completion + cost tracking are out of
-scope here (see COVERAGE.md).
+Mostly synchronous tier: a batch's completion window is 24h, so the lifecycle
+matrix never waits for "completed". Each case uploads a tiny JSONL, creates the
+batch through one of the four routing scenarios, asserts it was accepted
+(non-terminal status) and routed to the right provider, then retrieves / cancels /
+lists where the provider supports it. Everything created is deleted on teardown.
+The exception is TestBatchTerminalState, which carries completed-state + cost
+write-back coverage via a cross-run marker baton (design in COVERAGE.md).
 
 Routing signal: for provider_fallback the raw batch id discriminates the provider;
 for the encoded/unified/model_param scenarios the proxy re-encodes the id, so the
@@ -23,8 +24,9 @@ from datetime import datetime, timedelta, timezone
 from typing import Callable
 
 import pytest
+from pydantic import BaseModel
 
-from e2e_config import unique_marker
+from e2e_config import PROXY_BASE_URL, unique_marker
 
 from batch_client import (
     UPLOAD_FILENAME,
@@ -40,12 +42,17 @@ from capabilities import (
     BATCH_ID_SHAPE,
     CAPABILITIES,
     FILE_ID_SHAPE,
+    OPENAI_BATCH_BACKEND,
     OPENAI_BATCH_MODEL,
+    PROVIDERS,
     Capability,
+    Provider,
     batch_model_name,
     coverage_cells_for_lifecycle,
+    decoded_model_from_id,
     is_managed_id,
     matches_id_shape,
+    openai_batch_params,
     raw_id_matches_provider,
 )
 from e2e_http import (
@@ -474,11 +481,22 @@ def test_rate_limited_batch_create_leaves_no_unattributed_spend_row(
     )
 
 
-OPENAI_FILE_CONTENT_BACKEND = "gpt-4o-mini"
+FILE_CONTENT_CELLS = {
+    "azure": "llm.files.azure_openai.content.nonstream.works",
+    "vertex_ai": "llm.files.vertex.content.nonstream.works",
+    "bedrock": "llm.files.bedrock.content.nonstream.works",
+}
+BYTE_FIDELITY_CONTENT_PROVIDERS = frozenset({"azure"})
 
 
 class TestBatchFileContent:
-    """GET /v1/files/{id}/content returns the uploaded batch JSONL bytes."""
+    """GET /v1/files/{id}/content returns the uploaded batch JSONL bytes.
+
+    Azure stores the upload verbatim, so its download is asserted byte-equal.
+    Vertex (GCS) and Bedrock (S3) transform each JSONL line into the provider's
+    request format at upload time, so their downloads assert 200 plus non-empty
+    parseable JSON lines instead of byte equality.
+    """
 
     @pytest.mark.covers(
         "llm.files.openai.content.nonstream.works",
@@ -488,17 +506,11 @@ class TestBatchFileContent:
         self, client: BatchClient, resources: ResourceManager
     ) -> None:
         proxy_name = f"e2e-file-content-{unique_marker()}"
-        model_id = client.create_model(
-            proxy_name,
-            LiteLLMParamsBody(
-                model=f"openai/{OPENAI_FILE_CONTENT_BACKEND}",
-                api_key="os.environ/OPENAI_API_KEY",
-            ),
-        )
+        model_id = client.create_model(proxy_name, openai_batch_params())
         resources.defer(lambda: client.delete_model(model_id))
         key = resources.key()
 
-        payload = render_jsonl(OPENAI_FILE_CONTENT_BACKEND)
+        payload = render_jsonl(OPENAI_BATCH_BACKEND)
         file = unwrap(
             client.upload_file(
                 content=payload,
@@ -522,6 +534,62 @@ class TestBatchFileContent:
             "downloaded file content must match the uploaded JSONL bytes"
         )
 
+    @pytest.mark.parametrize(
+        "provider",
+        [
+            pytest.param(
+                p,
+                id=p.name,
+                marks=pytest.mark.covers(
+                    FILE_CONTENT_CELLS[p.name], exercised_on=["files"]
+                ),
+            )
+            for p in PROVIDERS
+            if p.name in FILE_CONTENT_CELLS
+        ],
+    )
+    def test_unified_file_content_downloads(
+        self,
+        provider: Provider,
+        client: BatchClient,
+        resources: ResourceManager,
+        batch_deployments: None,
+    ) -> None:
+        key = resources.key()
+        payload = render_jsonl(provider.raw_model)
+        file = unwrap(
+            client.upload_file(
+                content=payload,
+                form=FileUploadForm(purpose="batch", target_model_names=provider.model),
+                key=key,
+            )
+        )
+        resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
+        assert_file_object(file, provider=provider.name)
+        assert is_managed_id(file.id), (
+            f"{provider.name}: unified upload must return a managed file id, got {file.id!r}"
+        )
+
+        downloaded = client.proxy.transport.download(
+            f"/v1/files/{file.id}/content",
+            headers=client.proxy.transport.bearer(key),
+        )
+        assert downloaded.status_code == 200, (
+            f"{provider.name}: file content must be 200, "
+            f"got {downloaded.status_code}: {downloaded.body[:300]}"
+        )
+        body = downloaded.body.strip()
+        assert body, f"{provider.name}: file content download returned an empty body"
+        if provider.name in BYTE_FIDELITY_CONTENT_PROVIDERS:
+            assert body == payload.decode().strip(), (
+                f"{provider.name}: downloaded content must match the uploaded JSONL bytes"
+            )
+        else:
+            for line in body.splitlines():
+                assert json.loads(line), (
+                    f"{provider.name}: content line is not JSON: {line[:200]}"
+                )
+
 
 class TestOpenAIFiles:
     """GET /v1/files (list) and GET /v1/files/{id} (retrieve) over the OpenAI route.
@@ -1045,3 +1113,384 @@ class TestHostedVllmBatch:
             f"hosted_vllm batch has non-transitional status {batch.status!r}"
         )
         assert_batch_object(batch)
+
+
+BATCH_TERMINAL_STATUSES = frozenset({"completed", "failed", "expired", "cancelled"})
+FAILED_BATCH_POLL_SECONDS = 120.0
+FAILED_BATCH_POLL_INTERVAL_SECONDS = 5.0
+
+AZURE_BATCH_RAW_MODEL = next(p.raw_model for p in PROVIDERS if p.name == "azure")
+
+
+def _mismatched_endpoint_jsonl(model: str) -> bytes:
+    line = {
+        "custom_id": "req-1",
+        "method": "POST",
+        "url": "/v1/embeddings",
+        "body": {"model": model, "input": "ping"},
+    }
+    return (json.dumps(line) + "\n").encode()
+
+
+def _poll_until_terminal(client: BatchClient, batch_id: str, key: str) -> BatchObject:
+    deadline = time.monotonic() + FAILED_BATCH_POLL_SECONDS
+    fetched = retrieve_batch(client, batch_id, key=key, provider=None)
+    while fetched.status not in BATCH_TERMINAL_STATUSES and time.monotonic() < deadline:
+        time.sleep(FAILED_BATCH_POLL_INTERVAL_SECONDS)
+        fetched = retrieve_batch(client, batch_id, key=key, provider=None)
+    return fetched
+
+
+class TestBatchFailurePaths:
+    """Customer-facing failure contracts for /v1/batches.
+
+    A malformed input file is rejected at upload with a 400 naming the bad
+    content. A JSONL line whose url contradicts the batch endpoint is accepted
+    at create (providers validate asynchronously) and drives the batch to
+    "failed" with structured per-line errors, a null output_file_id, and a
+    zero-cost spend row (LIT-4852: a failed batch must book $0, not crash cost
+    tracking). Cancelling that already-failed batch returns a 409 naming the
+    terminal status. A file id encoded for one deployment wins over a
+    conflicting model param on create: the batch routes (and re-encodes) by the
+    file's embedded model, pinning that precedence.
+    """
+
+    @pytest.mark.covers(
+        "llm.batches.openai.malformed_jsonl.nonstream.works",
+        exercised_on=["files"],
+    )
+    def test_malformed_jsonl_upload_rejected(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        result = client.upload_file(
+            content=b"this is not json\n",
+            form=FileUploadForm(purpose="batch"),
+            model=OPENAI_BATCH_MODEL,
+            key=resources.key(),
+        )
+        match result:
+            case UnknownApiError(status_code=400, body=body):
+                assert "json" in body.lower(), (
+                    f"400 must name the malformed JSONL so users can fix the file, got: {body[:300]}"
+                )
+            case _:
+                pytest.fail(f"malformed JSONL upload must be rejected with a 400, got: {result}")
+
+    @pytest.mark.covers(
+        "llm.batches.openai.jsonl_endpoint_mismatch.nonstream.works",
+        "llm.batches.openai.cancel_terminal.nonstream.works",
+        exercised_on=["batches", "files"],
+    )
+    def test_endpoint_mismatch_fails_batch_and_cancel_conflicts(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        file = unwrap(
+            client.upload_file(
+                content=_mismatched_endpoint_jsonl("gpt-4o-mini"),
+                form=FileUploadForm(purpose="batch"),
+                model=OPENAI_BATCH_MODEL,
+                key=key,
+            )
+        )
+        resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
+
+        created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
+        require_successful_call(created)
+        batch = BatchObject.model_validate_json(created.body)
+
+        fetched = _poll_until_terminal(client, batch.id, key)
+        assert fetched.status == "failed", (
+            f"endpoint-mismatched batch must fail, got {fetched.status!r}"
+        )
+        assert fetched.output_file_id is None, (
+            f"failed batch must have no output file, got {fetched.output_file_id!r}"
+        )
+        assert fetched.errors is not None and fetched.errors.data, (
+            "failed batch must surface structured errors so users can fix the JSONL"
+        )
+        first_error = fetched.errors.data[0]
+        assert first_error.message, "batch error item has no message"
+        assert first_error.code, "batch error item has no code"
+
+        rows = client.proxy.poll_logs_for_request_id(f"{fetched.id}_batch_cost")
+        assert rows, (
+            f"failed batch {fetched.id} wrote no spend row; retrieve must book $0 (LIT-4852)"
+        )
+        assert all((row.spend or 0) == 0 for row in rows), (
+            f"failed batch must cost $0, got {[(r.request_id, r.spend) for r in rows]}"
+        )
+        assert rows[0].call_type == "aretrieve_batch", (
+            f"batch cost row call_type={rows[0].call_type!r}"
+        )
+
+        conflict = client.cancel_batch(batch.id, key=key)
+        match conflict:
+            case UnknownApiError(status_code=409, body=body):
+                assert "failed" in body.lower(), (
+                    f"409 must name the terminal status blocking the cancel, got: {body[:300]}"
+                )
+            case _:
+                pytest.fail(f"cancel of a failed batch must return a 409 conflict, got: {conflict}")
+
+    @pytest.mark.covers(
+        "llm.batches.openai.foreign_file_id.nonstream.works",
+        exercised_on=["batches", "files"],
+    )
+    def test_foreign_encoded_file_id_routes_by_file_model(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        file = unwrap(
+            client.upload_file(
+                content=render_jsonl(AZURE_BATCH_RAW_MODEL),
+                form=FileUploadForm(purpose="batch"),
+                model=AZURE_BATCH_MODEL,
+                key=key,
+            )
+        )
+        resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
+        assert decoded_model_from_id(file.id) == AZURE_BATCH_MODEL, (
+            f"upload did not encode the azure deployment into the file id: {file.id!r}"
+        )
+
+        created = client.create_batch(
+            body=BatchCreateBody(input_file_id=file.id, model=OPENAI_BATCH_MODEL), key=key
+        )
+        require_successful_call(created)
+        batch = BatchObject.model_validate_json(created.body)
+        resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key)))
+
+        assert decoded_model_from_id(batch.id) == AZURE_BATCH_MODEL, (
+            "create with a foreign encoded file id must route by the file's embedded model, "
+            f"but the batch id encodes {decoded_model_from_id(batch.id)!r} "
+            f"(model param was {OPENAI_BATCH_MODEL!r})"
+        )
+        fetched = retrieve_batch(client, batch.id, key=key, provider=None)
+        assert fetched.id == batch.id
+        assert fetched.status, "retrieved foreign-file batch has no status"
+
+
+class TestBatchSecondHop:
+    """Two-proxy batch routing: a litellm_proxy deployment chained to the gateway
+    itself (LIT-5347, PR #36240).
+
+    The hop deployment's litellm_params point litellm_proxy/<inner model> at this
+    gateway's own base URL with a freshly minted virtual key, so the unified
+    upload and batch create traverse gateway -> gateway -> OpenAI. The regression
+    this pins: target_model_names must be rewritten to the inner deployment on
+    the second hop and the nested managed ids must round-trip retrieve.
+    """
+
+    @pytest.mark.covers(
+        "llm.batches.openai.second_hop.nonstream.works",
+        exercised_on=["batches", "files"],
+    )
+    def test_unified_create_and_retrieve_via_chained_gateway(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        hop_name = batch_model_name("openai-batch-hop")
+        model_id = client.create_model(
+            hop_name,
+            LiteLLMParamsBody(
+                model=f"litellm_proxy/{OPENAI_BATCH_MODEL}",
+                api_base=PROXY_BASE_URL,
+                api_key=key,
+            ),
+        )
+        resources.defer(lambda: client.delete_model(model_id))
+
+        file = unwrap(
+            client.upload_file(
+                content=render_jsonl("gpt-4o-mini"),
+                form=FileUploadForm(purpose="batch", target_model_names=hop_name),
+                key=key,
+            )
+        )
+        resources.defer(quietly(lambda: client.delete_file(file.id, key=key)))
+        assert is_managed_id(file.id), (
+            f"second-hop unified upload must return a managed file id, got {file.id!r}"
+        )
+
+        created = client.create_batch(body=BatchCreateBody(input_file_id=file.id), key=key)
+        require_successful_call(created)
+        batch = BatchObject.model_validate_json(created.body)
+        resources.defer(quietly(lambda: client.cancel_batch(batch.id, key=key)))
+
+        assert is_managed_id(batch.id), (
+            f"second-hop create must return a managed batch id, got {batch.id!r}"
+        )
+        assert batch.status in CREATED_BATCH_STATUSES, (
+            f"second-hop batch has non-transitional status {batch.status!r}"
+        )
+        assert_batch_object(batch)
+
+        fetched = retrieve_batch(client, batch.id, key=key, provider=None)
+        assert fetched.id == batch.id
+        assert fetched.status, "second-hop retrieve returned no status"
+
+
+class BatchOutputBody(BaseModel):
+    choices: list[object] = []
+
+
+class BatchOutputResponse(BaseModel):
+    status_code: int | None = None
+    body: BatchOutputBody | None = None
+
+
+class BatchOutputLine(BaseModel):
+    response: BatchOutputResponse
+
+
+TERMINAL_MARKER_KEY = "litellm_e2e_suite"
+TERMINAL_MARKER_VALUE = "batches-terminal-baton"
+TERMINAL_POLL_SECONDS = 300.0
+TERMINAL_POLL_INTERVAL_SECONDS = 10.0
+TERMINAL_LIST_LIMIT = 100
+TERMINAL_BAND_MIN_AGE_SECONDS = 25 * 3600
+TERMINAL_BAND_MAX_AGE_SECONDS = 73 * 3600
+
+
+def _marker_batches(client: BatchClient, key: str) -> list[BatchObject]:
+    listed = unwrap(
+        client.list_batches(key=key, model=OPENAI_BATCH_MODEL, limit=TERMINAL_LIST_LIMIT)
+    )
+    return [
+        b
+        for b in listed.data
+        if (b.metadata or {}).get(TERMINAL_MARKER_KEY) == TERMINAL_MARKER_VALUE
+    ]
+
+
+def _await_completed_marker(
+    client: BatchClient, key: str
+) -> tuple[BatchObject | None, list[BatchObject]]:
+    deadline = time.monotonic() + TERMINAL_POLL_SECONDS
+    while True:
+        markers = _marker_batches(client, key)
+        completed = max(
+            (b for b in markers if b.status == "completed"),
+            key=lambda b: b.created_at or 0,
+            default=None,
+        )
+        if completed is not None or time.monotonic() >= deadline:
+            return completed, markers
+        time.sleep(TERMINAL_POLL_INTERVAL_SECONDS)
+
+
+def _assert_aged_markers_terminal(markers: list[BatchObject]) -> None:
+    now = time.time()
+    stuck = [
+        b
+        for b in markers
+        if b.created_at is not None
+        and TERMINAL_BAND_MIN_AGE_SECONDS <= now - b.created_at <= TERMINAL_BAND_MAX_AGE_SECONDS
+        and b.status not in BATCH_TERMINAL_STATUSES
+    ]
+    assert not stuck, (
+        "marker batches past their 24h completion window must be terminal; stuck: "
+        f"{[(b.id, b.status, b.created_at) for b in stuck]}"
+    )
+
+
+class TestBatchTerminalState:
+    """Terminal state + cost write-back via a cross-run marker baton.
+
+    Each run submits a 1-line marker batch (stable metadata key/value plus a
+    per-run field) and never cancels or deletes it: the marker is the baton the
+    next run picks up. Polling is list-only for up to 5 minutes because a
+    retrieve of a non-terminal batch books a $0 spend row whose request_id then
+    blocks the real-cost row (skip_duplicates); the single retrieve happens only
+    once a completed marker exists. The assertion target is the newest completed
+    marker from ANY run, so on the 6h stage cadence the full assertions are
+    deterministic from run 2 onward. On a cold start (no marker has ever
+    completed within the poll budget) the test passes on the submission
+    assertions alone: that is a documented vacuous pass, not a skip, and this
+    run's marker becomes the next run's target. Markers aged past OpenAI's 24h
+    completion window (25h-73h band, within the newest list page) must be
+    terminal. The cost assertion is the LIT-5730 headline: retrieving a
+    completed model-encoded batch must write a positive spend row keyed
+    {batch_id}_batch_cost; before the fix the logging worker fetched the
+    re-encoded output_file_id, 404d, and the row never landed.
+    """
+
+    @pytest.mark.covers(
+        "llm.batches.openai.terminal_state.nonstream.works",
+        "llm.batches.openai.terminal_state.nonstream.cost_logged",
+        exercised_on=["batches", "files"],
+    )
+    def test_completed_batch_downloads_output_and_books_cost(
+        self, client: BatchClient, resources: ResourceManager, batch_deployments: None
+    ) -> None:
+        key = resources.key()
+        file = unwrap(
+            client.upload_file(
+                content=render_jsonl("gpt-4o-mini"),
+                form=FileUploadForm(purpose="batch"),
+                model=OPENAI_BATCH_MODEL,
+                key=key,
+            )
+        )
+        created = client.create_batch(
+            body=BatchCreateBody(
+                input_file_id=file.id,
+                metadata={
+                    TERMINAL_MARKER_KEY: TERMINAL_MARKER_VALUE,
+                    "run": unique_marker(),
+                },
+            ),
+            key=key,
+        )
+        require_successful_call(created)
+        submitted = BatchObject.model_validate_json(created.body)
+        assert submitted.status in CREATED_BATCH_STATUSES, (
+            f"marker batch has non-transitional status {submitted.status!r}"
+        )
+        assert (submitted.metadata or {}).get(TERMINAL_MARKER_KEY) == TERMINAL_MARKER_VALUE, (
+            f"create dropped the marker metadata: {submitted.metadata!r}"
+        )
+
+        completed, markers = _await_completed_marker(client, key)
+        _assert_aged_markers_terminal(markers)
+        if completed is None:
+            return
+
+        fetched = retrieve_batch(client, completed.id, key=key, provider=None)
+        assert fetched.status == "completed", (
+            f"listed-completed marker retrieved as {fetched.status!r}"
+        )
+        assert fetched.output_file_id, "completed batch has no output_file_id"
+
+        downloaded = client.proxy.transport.download(
+            f"/v1/files/{fetched.output_file_id}/content",
+            headers=client.proxy.transport.bearer(key),
+        )
+        assert downloaded.status_code == 200, (
+            f"output content must be 200, got {downloaded.status_code}: {downloaded.body[:300]}"
+        )
+        first_line = BatchOutputLine.model_validate_json(downloaded.body.strip().splitlines()[0])
+        assert first_line.response.status_code == 200, (
+            f"batch output line reports failure: {downloaded.body[:400]}"
+        )
+        assert first_line.response.body is not None and first_line.response.body.choices, (
+            "batch output line has no choices"
+        )
+
+        rows = client.proxy.poll_logs_for_request_id(
+            f"{fetched.id}_batch_cost",
+            predicate=lambda found: any((row.spend or 0) > 0 for row in found),
+        )
+        priced = [row for row in rows if (row.spend or 0) > 0]
+        assert priced, (
+            f"completed batch {fetched.id} wrote no positive-cost spend row under "
+            f"request_id {fetched.id}_batch_cost; cost write-back is broken (LIT-5730)"
+        )
+        cost_row = priced[0]
+        assert cost_row.call_type == "aretrieve_batch", (
+            f"batch cost row call_type={cost_row.call_type!r}"
+        )
+        assert (cost_row.total_tokens or 0) > 0, (
+            f"batch cost row has no token usage: {cost_row.total_tokens!r}"
+        )

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -64,6 +64,7 @@
 - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
 - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}
 - {id: llm.responses.openai.passthrough.stream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [cost_logged], source: "test_passthrough_e2e.py", rationale: "A streamed POST /openai_passthrough/v1/responses is costed and keyed by the provider response id; it used to log a zero-cost row under a random id (GitHub issue #36523)"}
+- {id: llm.responses.openai.passthrough_websocket.stream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai/v1/responses is accepted, so a responses.connect client reaches OpenAI through the same prefix its HTTP traffic uses; the prefix carried no websocket route and refused the upgrade with a 403 (GitHub issue #36088)"}
 - {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"}
 - {id: llm.responses.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Vision via Responses API"}
 - {id: llm.responses.anthropic.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Anthropic translation (smoke)"}

tests/e2e/coverage_registry/llm_nonconversational.yaml

@@ -26,6 +26,13 @@
 - {id: llm.batches.hosted_vllm.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible batch create"}
 - {id: llm.batches.openai.key_model_access_denied.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "Key model restriction 403 on upload/create"}
 - {id: llm.batches.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.18 / LIT-4778", rationale: "Missing input_file_id and invalid batch id rejected"}
+- {id: llm.batches.openai.terminal_state.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "A batch actually reaches completed and its output file downloads through GET /v1/files/{id}/content with per-line provider responses"}
+- {id: llm.batches.openai.terminal_state.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "test_batches_e2e.py / LIT-5730", fail_before_fix: proven, rationale: "Retrieving a completed model-encoded batch writes a positive spend row keyed {batch_id}_batch_cost (pins LIT-4852/LIT-5666; before the fix the logging worker 404d fetching the re-encoded output_file_id and the row was never written)"}
+- {id: llm.batches.openai.malformed_jsonl.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Uploading a non-JSON batch file is rejected with a 400 naming the bad line"}
+- {id: llm.batches.openai.jsonl_endpoint_mismatch.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "JSONL line url that contradicts the batch endpoint drives the batch to failed with structured errors, retrieve stays clean, and the terminal retrieve books a zero-cost spend row (LIT-4852)"}
+- {id: llm.batches.openai.cancel_terminal.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Cancelling an already-terminal batch returns a 409 conflict naming the terminal status"}
+- {id: llm.batches.openai.foreign_file_id.nonstream.works, module: llm, tier: P1, subject_endpoint: batches, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "Create with one deployment's encoded file id and a conflicting model param routes by the file's embedded model; the returned batch id pins that precedence"}
+- {id: llm.batches.openai.second_hop.nonstream.works, module: llm, tier: P0, subject_endpoint: batches, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5347", rationale: "A litellm_proxy deployment chained to the gateway itself preserves target_model_names through nested unified ids; upload, create, and retrieve work over the two-hop chain (PR #36240)"}
 - {id: llm.files.openai.upload.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "openai_files_endpoints/files_endpoints.py:46", rationale: "File upload returns OpenAIFileObject"}
 - {id: llm.files.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.16 / LIT-4778", rationale: "File upload without purpose rejected"}
 - {id: llm.files.openai.retrieve.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "files_endpoints.py", rationale: "File retrieve by id"}
@@ -40,10 +47,14 @@
 - {id: llm.files.hosted_vllm.upload.nonstream.works, module: llm, tier: P1, subject_endpoint: files, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "hosted_vllm OpenAI-compatible file upload"}
 - {id: llm.rerank.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: rerank, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_rerank_e2e.py:29", rationale: "Cohere rerank, top_n + relevance_score"}
 - {id: llm.files.openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py", rationale: "GET /v1/files/{id}/content returns uploaded batch JSONL bytes"}
+- {id: llm.files.azure_openai.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on an Azure unified file returns the uploaded JSONL bytes verbatim"}
+- {id: llm.files.vertex.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on a Vertex unified file streams the GCS object back (provider-transformed JSONL, so asserts non-empty JSON lines rather than byte equality)"}
+- {id: llm.files.bedrock.content.nonstream.works, module: llm, tier: P0, subject_endpoint: files, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "test_batches_e2e.py / LIT-5730", rationale: "GET /v1/files/{id}/content on a Bedrock unified file streams the S3 object back (provider-transformed JSONL, so asserts non-empty JSON lines rather than byte equality)"}
 - {id: llm.realtime.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "test_realtime_bedrock_e2e.py", rationale: "Nova Sonic realtime session emits response.done (LIT-2239)"}
 - {id: llm.google_native.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "LIT-4076 / proxy/google_endpoints/endpoints.py", fail_before_fix: proven, rationale: "google-native generateContent must stamp x-litellm-response-cost so SDK traffic reconciles against spend"}
 - {id: llm.google_native.gemini.basic.stream.works, module: llm, tier: P0, subject_endpoint: google_native, route: gemini, capability: basic, streaming: stream, assertions: [works], source: "PR #28213 / proxy/proxy_server.py async_data_generator", fail_before_fix: proven, rationale: "streamGenerateContent must relay single-prefixed SSE frames with no [DONE] sentinel; doubled data: prefixes and the OpenAI terminator both break the Vertex Java SDK"}
 - {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets returns an ephemeral credential"}
+- {id: llm.realtime.openai.passthrough.stream.works, module: llm, tier: P0, subject_endpoint: realtime, route: openai, capability: basic, streaming: stream, assertions: [works], fail_before_fix: proven, source: "test_passthrough_e2e.py", rationale: "A websocket upgrade on /openai_passthrough/v1/realtime is accepted and relayed to OpenAI; only HTTP routes were registered under the prefix, so realtime clients were refused with a 403 before a socket existed (GitHub issue #36088)"}
 - {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"}
 - {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"}
 - {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"}

tests/e2e/e2e_config.py

@@ -150,6 +150,15 @@ ANOMALY_SPEND_SETTLE_SECONDS = float(
 )
 
 
+def ws_base_url() -> str:
+    """PROXY_BASE_URL with its scheme swapped for the websocket one, so a suite
+    opening a socket points at the same proxy every HTTP suite uses."""
+    for scheme, ws_scheme in (("https://", "wss://"), ("http://", "ws://")):
+        if PROXY_BASE_URL.startswith(scheme):
+            return ws_scheme + PROXY_BASE_URL[len(scheme) :]
+    return PROXY_BASE_URL
+
+
 def datadog_mcp_url(*, toolsets: str = "core") -> str:
     """Regional Datadog remote MCP endpoint for this process's DD_SITE.
 

tests/e2e/fixture_bundle.py

@@ -5,7 +5,9 @@ version + format version) plus one subdirectory per test, holding one JSON file
 per provider-bound interaction in call order. Bundles older than
 ``MAX_BUNDLE_AGE`` hard-fail replay at collection time (see conftest), so a
 green replay run can never certify against fixtures that have drifted more than
-a week from the live providers.
+a week from the live providers. Bump ``BUNDLE_FORMAT_VERSION`` whenever a change
+moves recorded keys: a bundle recorded under the old rules then fails naming
+both versions instead of quietly missing on every call.
 
 This module owns the format only. The provider-edge server that produces and
 consumes it lives in provider_edge.py (LIT-5745) and the canonical match keys
@@ -28,7 +30,7 @@ from typing import Final
 
 from pydantic import BaseModel, JsonValue
 
-BUNDLE_FORMAT_VERSION: Final = 2
+BUNDLE_FORMAT_VERSION: Final = 3
 MAX_BUNDLE_AGE: Final = timedelta(days=7)
 MANIFEST_FILENAME: Final = "manifest.json"
 
@@ -47,7 +49,14 @@ class RecordedRequest(BaseModel):
     over ``method``, ``path`` (the edge path including the provider mount,
     query string excluded), and the canonicalized headers, params, body, form,
     and file identity. Non-JSON bodies store a canonicalized content digest
-    instead of the bytes."""
+    instead of the bytes.
+
+    ``file_name`` is a JSON list of the uploaded parts' ``[field, filename,
+    content-type]`` triples rather than a flat label, so a separator inside a
+    filename cannot impersonate a field boundary. ``file_bytes`` is recorded for
+    a reader's benefit and stays out of the key: the canonicalizer absorbs
+    timestamp and id drift inside an uploaded file, and that drift moves the
+    byte count."""
 
     method: str
     path: str

tests/e2e/fixture_canonical.py

@@ -129,7 +129,6 @@ def canonicalize(request: RecordedRequest) -> CanonicalRequest:
         else {
             "name": None if request.file_name is None else canonical_string(request.file_name),
             "sha256": request.file_sha256,
-            "bytes": request.file_bytes,
         }
     )
     content: Final[dict[str, JsonValue]] = {

tests/e2e/llm_translation/endpoints_client.py

@@ -87,6 +87,7 @@ class RichMessagesRequest(BaseModel):
     max_tokens: int = 64
     system: list[TextBlock]
     messages: list[RichMessage]
+    cache: dict[str, bool] = {"no-cache": True}
 
 
 class CompletionsRequest(BaseModel):

tests/e2e/llm_translation/passthrough_client.py

@@ -11,9 +11,13 @@ native request models are co-located here because only this suite uses them.
 from __future__ import annotations
 
 from dataclasses import dataclass
+from urllib.parse import urlencode
 
 from pydantic import BaseModel, Field
+from websockets.exceptions import InvalidStatus
+from websockets.sync.client import connect
 
+from e2e_config import ws_base_url
 from proxy_client import ProxyClient
 from e2e_http import FileUploadForm, Headers, NoBody, Result, StreamingResponse
 from models import ChatMessage
@@ -175,6 +179,26 @@ class OpenAIEmbeddingBody(BaseModel):
     input: str
 
 
+class WebsocketEnvelope(BaseModel):
+    """The one field every provider event carries, so the first frame off a
+    passthrough socket identifies itself without the suite parsing raw dicts."""
+
+    type: str
+
+
+class WebsocketHandshake(BaseModel):
+    """What the proxy did with a websocket upgrade on a passthrough prefix.
+
+    `rejected_status` is the HTTP status of a refused upgrade: a prefix carrying no
+    websocket route answers 403, before any socket exists. `first_event_type` is the
+    type of the first frame an accepted socket delivered, which is None when the
+    provider waits for the client to speak first.
+    """
+
+    rejected_status: int | None = None
+    first_event_type: str | None = None
+
+
 class PassthroughBatchList(BaseModel):
     """OpenAI's own batch page, relayed verbatim. `object` is required so a body
     that is not an OpenAI list fails validation instead of passing vacuously."""
@@ -339,5 +363,39 @@ class PassthroughClient:
             ),
         )
 
+    # ---- OpenAI websocket passthrough ----------------------------------
+    #
+    # The same prefixes over an upgrade instead of a POST, for the provider APIs
+    # that only speak websocket (realtime, responses.connect).
+
+    def openai_passthrough_websocket(
+        self,
+        key: str,
+        path: str,
+        *,
+        model: str | None = None,
+        open_timeout: float = 30.0,
+        first_event_timeout: float = 30.0,
+    ) -> WebsocketHandshake:
+        query = f"?{urlencode({'model': model})}" if model is not None else ""
+        try:
+            connection = connect(
+                f"{ws_base_url()}{path}{query}",
+                additional_headers={"Authorization": f"Bearer {key}"},
+                open_timeout=open_timeout,
+            )
+        except InvalidStatus as rejected:
+            return WebsocketHandshake(rejected_status=rejected.response.status_code)
+        with connection:
+            try:
+                frame = connection.recv(timeout=first_event_timeout)
+            except TimeoutError:
+                return WebsocketHandshake()
+            text = frame.decode("utf-8") if isinstance(frame, bytes) else frame
+            return WebsocketHandshake(
+                first_event_type=WebsocketEnvelope.model_validate_json(text).type
+            )
+
+
 def build_client(proxy: ProxyClient) -> PassthroughClient:
     return PassthroughClient(proxy=proxy)