Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_fix_v1_model_info

.circleci/config.yml

@@ -2690,6 +2690,122 @@ jobs:
           path: ui/litellm-dashboard/playwright-report
           destination: e2e-playwright-report
 
+  e2e_ui_testing_server_root_path:
+    docker:
+      - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
+        environment:
+          POSTGRES_USER: e2euser
+          POSTGRES_PASSWORD: e2epassword
+          POSTGRES_DB: litellm_e2e
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
+      CI: "true"
+      # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH
+      # is read both by the proxy at boot (to rewrite the built UI bundle in place)
+      # and by migration.serverRootPath.config.ts, which refuses to run without it.
+      SERVER_ROOT_PATH: "/litellm"
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - restore_cache:
+          keys:
+            - v1-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Python dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+            uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
+      - save_cache:
+          key: v1-uv-cache-{{ checksum "uv.lock" }}
+          paths:
+            - ~/.cache/uv
+      - restore_cache:
+          keys:
+            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+      - run:
+          name: Install Node dependencies and Playwright
+          command: |
+            cd ui/litellm-dashboard
+            npm ci
+            npx playwright install chromium
+      - save_cache:
+          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          paths:
+            - ui/litellm-dashboard/node_modules
+            - ~/.cache/ms-playwright
+      - run:
+          name: Build UI from source
+          command: |
+            cd ui/litellm-dashboard
+            npm run build
+            rm -rf ../../litellm/proxy/_experimental/out
+            mv out ../../litellm/proxy/_experimental/out
+            find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
+              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
+            done
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "30"
+      - run:
+          name: Push Prisma schema
+          command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+      - run:
+          name: Seed database
+          command: |
+            PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
+              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+      - run:
+          name: Start mock LLM server
+          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          background: true
+      - run:
+          name: Start LiteLLM proxy under a server root path
+          environment:
+            LITELLM_MASTER_KEY: "sk-1234"
+            MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
+            DISABLE_SCHEMA_UPDATE: "true"
+          # Output flows to this step's own log, so a boot crash is visible here
+          # rather than swallowed by a downstream readiness probe.
+          command: |
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              uv run --no-sync python -m litellm.proxy.proxy_cli \
+                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --port 4000
+          background: true
+      - run:
+          name: Wait for prefixed proxy to be ready
+          command: |
+            for i in $(seq 1 60); do
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
+              if [ "$HTTP_CODE" = "200" ]; then
+                echo "Prefixed proxy is ready"
+                exit 0
+              fi
+              sleep 2
+            done
+            echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log"
+            exit 1
+      - run:
+          name: Run migration smoke under SERVER_ROOT_PATH
+          command: |
+            cd ui/litellm-dashboard
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
+          no_output_timeout: 10m
+      - store_artifacts:
+          path: ui/litellm-dashboard/test-results
+          destination: e2e-server-root-path-test-results
+      - store_artifacts:
+          path: ui/litellm-dashboard/playwright-report
+          destination: e2e-server-root-path-playwright-report
+
   build_docker_database_image:
     machine:
       image: ubuntu-2204:2024.04.1
@@ -2795,6 +2911,8 @@ workflows:
           filters: *main_branches
       - e2e_ui_testing:
           filters: *main_branches
+      - e2e_ui_testing_server_root_path:
+          filters: *main_branches
       - build_and_test:
           requires:
             - build_docker_database_image

litellm/anthropic_beta_headers_config.json

@@ -129,7 +129,7 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": null,
-    "compact-2026-01-12": null,
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",

litellm/caching/caching.py

@@ -691,6 +691,7 @@ class Cache:
         self,
         embedding_response: Any,
         model: Optional[str],
+        prompt_tokens: Optional[int] = None,
         prompt_tokens_details: Optional[dict] = None,
     ) -> CachedEmbedding:
         """
@@ -703,6 +704,7 @@ class Cache:
                     "index": embedding_response.get("index"),
                     "object": embedding_response.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
             elif hasattr(embedding_response, "model_dump"):
@@ -712,6 +714,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
             else:
@@ -721,6 +724,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
         except KeyError as e:
@@ -769,6 +773,29 @@ class Cache:
                 per_item[key] = value
         return per_item if per_item else None
 
+    def _get_per_item_prompt_tokens(
+        self,
+        result: EmbeddingResponse,
+        idx_in_result_data: int,
+    ) -> Optional[int]:
+        """
+        Extract the per-item prompt_tokens from a response for caching.
+
+        Single-item responses store the full usage.prompt_tokens. Multi-item
+        responses distribute it evenly (with remainder) so that summing all
+        per-item values on retrieval reconstructs the original total.
+        """
+        if result.usage is None or result.usage.prompt_tokens is None:
+            return None
+
+        total = result.usage.prompt_tokens
+        num_items = len(result.data)
+        if num_items <= 1:
+            return total
+
+        quotient, remainder = divmod(total, num_items)
+        return quotient + (1 if idx_in_result_data < remainder else 0)
+
     def add_embedding_response_to_cache(
         self,
         result: EmbeddingResponse,
@@ -780,7 +807,11 @@ class Cache:
         kwargs["cache_key"] = preset_cache_key
         embedding_response = result.data[idx_in_result_data]
 
-        # Extract per-item prompt_tokens_details from response usage
+        # Extract per-item prompt_tokens + details from response usage
+        prompt_tokens = self._get_per_item_prompt_tokens(
+            result=result,
+            idx_in_result_data=idx_in_result_data,
+        )
         prompt_tokens_details = self._get_per_item_prompt_tokens_details(
             result=result,
             idx_in_result_data=idx_in_result_data,
@@ -791,6 +822,7 @@ class Cache:
         embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
             embedding_response,
             model_name,
+            prompt_tokens=prompt_tokens,
             prompt_tokens_details=prompt_tokens_details,
         )
 

litellm/caching/caching_handler.py

@@ -394,7 +394,7 @@ class LLMCachingHandler:
                 return cr["model"]
         return None
 
-    def _process_async_embedding_cached_response(
+    def _process_async_embedding_cached_response(  # noqa: PLR0915
         self,
         final_embedding_cached_response: Optional[EmbeddingResponse],
         cached_result: List[Optional[CachedEmbedding]],
@@ -456,7 +456,10 @@ class LLMCachingHandler:
                             index=idx,
                             object="embedding",
                         )
-                    if isinstance(kwargs_input_as_list[idx], str):
+                    cached_prompt_tokens = cr.get("prompt_tokens")
+                    if cached_prompt_tokens is not None:
+                        prompt_tokens += cached_prompt_tokens
+                    elif isinstance(kwargs_input_as_list[idx], str):
                         from litellm.utils import token_counter
 
                         prompt_tokens += token_counter(

litellm/constants.py

@@ -1158,6 +1158,7 @@ BEDROCK_CONVERSE_MODELS = [
     "openai.gpt-oss-120b-1:0",
     "anthropic.claude-haiku-4-5-20251001-v1:0",
     "anthropic.claude-sonnet-4-5-20250929-v1:0",
+    "anthropic.claude-fable-5",
     "anthropic.claude-opus-4-8",
     "anthropic.claude-opus-4-7",
     "anthropic.claude-opus-4-6-v1:0",

litellm/litellm_core_utils/cli_token_utils.py

@@ -37,7 +37,7 @@ def get_litellm_gateway_api_key(
     """
     Get the stored CLI API key for use with LiteLLM SDK.
 
-    This function reads the token file created by `litellm-proxy login`
+    This function reads the token file created by `lite login`
     and returns the API key for use in Python scripts.
 
     Args:

litellm/litellm_core_utils/get_litellm_params.py

@@ -34,6 +34,7 @@ _OPTIONAL_KWARGS_KEYS = frozenset(
         "aws_bedrock_runtime_endpoint",
         "tpm",
         "rpm",
+        "use_xai_oauth",
     }
 )
 

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -633,11 +633,6 @@ def convert_to_model_response_object(  # noqa: PLR0915
                         thinking_blocks = choice["message"]["thinking_blocks"]
                         provider_specific_fields["thinking_blocks"] = thinking_blocks
 
-                    if reasoning_content:
-                        provider_specific_fields["reasoning_content"] = (
-                            reasoning_content
-                        )
-
                     message = Message(
                         content=content,
                         role=choice["message"]["role"] or "assistant",

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -4290,6 +4290,49 @@ def _deduplicate_bedrock_tool_content(
     return _deduplicate_bedrock_content_blocks(tool_content, "toolResult")
 
 
+def _rename_duplicate_bedrock_document_names(
+    contents: List[BedrockMessageBlock],
+) -> List[BedrockMessageBlock]:
+    """
+    Rename duplicate document names across all messages in a Bedrock request.
+
+    Document names are derived from a content hash, so the same file appearing
+    in multiple conversation turns produces identical names and Bedrock rejects
+    the request with "Messages can not contain duplicate document names".  The
+    first occurrence keeps its original name so prompt-cache prefixes stay
+    stable; later occurrences get a deterministic positional suffix
+    (``_2``, ``_3``, ...), bumped further if the suffixed name already
+    belongs to another document (e.g. an organic name ending in ``_2``).
+    """
+    used_names: Set[str] = set()
+    for message in contents:
+        for block in message.get("content") or []:
+            document = block.get("document")
+            if isinstance(document, dict) and document.get("name"):
+                used_names.add(document["name"])
+
+    name_counts: Dict[str, int] = {}
+    for message in contents:
+        for block in message.get("content") or []:
+            document = block.get("document")
+            if not isinstance(document, dict):
+                continue
+            name = document.get("name")
+            if not name:
+                continue
+            count = name_counts.get(name, 0) + 1
+            name_counts[name] = count
+            if count > 1:
+                suffix = count
+                new_name = f"{name}_{suffix}"
+                while new_name in used_names:
+                    suffix += 1
+                    new_name = f"{name}_{suffix}"
+                used_names.add(new_name)
+                document["name"] = new_name
+    return contents
+
+
 def _sort_bedrock_assistant_content_blocks(
     blocks: List[BedrockContentBlock],
 ) -> List[BedrockContentBlock]:
@@ -4938,7 +4981,7 @@ class BedrockConverseMessagesProcessor:
                     llm_provider=llm_provider,
                 )
 
-        return contents
+        return _rename_duplicate_bedrock_document_names(contents)
 
     @staticmethod
     def translate_thinking_blocks_to_reasoning_content_blocks(
@@ -5360,7 +5403,7 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                 llm_provider=llm_provider,
             )
 
-    return contents
+    return _rename_duplicate_bedrock_document_names(contents)
 
 
 def make_valid_bedrock_tool_name(input_tool_name: str) -> str:

litellm/llms/anthropic/chat/transformation.py

@@ -1455,10 +1455,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 _value = self._map_stop_sequences(value)
                 if _value is not None:
                     optional_params["stop_sequences"] = _value
-            elif param == "temperature":
-                optional_params["temperature"] = value
-            elif param == "top_p":
-                optional_params["top_p"] = value
+            elif param == "temperature" or param == "top_p":
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=optional_params,
+                    model=model,
+                    param=param,
+                    value=value,
+                    drop_params=drop_params,
+                    output_key=param,
+                )
             elif param == "response_format" and isinstance(value, dict):
                 if any(
                     substring in model
@@ -1975,6 +1980,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         optional_params.pop("is_vertex_request", None)
         optional_params.pop("client_metadata", None)
 
+        # ``top_k`` is a provider-specific kwarg that bypasses
+        # ``map_openai_params``; gate it here, the single boundary shared by
+        # the direct Anthropic, Bedrock invoke, Vertex, and Azure paths.
+        top_k = optional_params.pop("top_k", None)
+        if top_k is not None:
+            AnthropicConfig._apply_sampling_param(
+                optional_params=optional_params,
+                model=model,
+                param="top_k",
+                value=top_k,
+                drop_params=litellm_params.get("drop_params") is True,
+                output_key="top_k",
+            )
+
         data = {
             "model": model,
             "messages": anthropic_messages,

litellm/llms/anthropic/common_utils.py

@@ -272,23 +272,68 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         )
 
     @staticmethod
-    def _supports_model_capability(model: str, key: str) -> bool:
-        """Check a boolean capability ``key`` in the model map.
+    def _supports_sampling_params(model: str) -> bool:
+        """Claude 4.7+ (Opus 4.7/4.8, Fable 5) removed sampling params: the API
+        rejects ``top_p``, ``top_k``, and any ``temperature`` other than 1 with
+        a 400 ("`temperature` is deprecated for this model").
 
-        Strips bedrock/vertex prefixes so a provider-routed Claude still
-        resolves to the Anthropic model-map entry.
-        """
-        from litellm.utils import _supports_factory
+        Driven by the ``supports_sampling_params`` flag in the model map; the
+        name check remains only as a fallback for provider-routed ids whose
+        map entries predate the flag."""
+        flag = AnthropicModelInfo._get_model_capability(
+            model, "supports_sampling_params"
+        )
+        if flag is not None:
+            return flag
+        model_lower = model.lower()
+        return not any(
+            v in model_lower
+            for v in (
+                "fable",
+                "opus-4-7",
+                "opus_4_7",
+                "opus-4.7",
+                "opus_4.7",
+                "opus-4-8",
+                "opus_4_8",
+                "opus-4.8",
+                "opus_4.8",
+            )
+        )
 
-        try:
-            if _supports_factory(
-                model=model,
-                custom_llm_provider="anthropic",
-                key=key,
-            ):
-                return True
-        except Exception:
-            pass
+    @staticmethod
+    def _apply_sampling_param(
+        optional_params: dict,
+        model: str,
+        param: str,
+        value: Any,
+        drop_params: bool,
+        output_key: str,
+    ) -> None:
+        """Forward ``temperature``/``top_p``/``top_k`` to
+        ``optional_params[output_key]`` unless the model removed sampling
+        params, in which case drop the param (with drop_params) or raise a
+        clean client-side 400."""
+        if AnthropicModelInfo._supports_sampling_params(model) or (
+            param == "temperature" and value == 1
+        ):
+            optional_params[output_key] = value
+        elif not (litellm.drop_params or drop_params):
+            supported_hint = (
+                "Only temperature=1 is supported. " if param == "temperature" else ""
+            )
+            raise litellm.utils.UnsupportedParamsError(
+                message=(
+                    f"{model} does not support {param}={value}. {supported_hint}"
+                    "To drop unsupported params, set `litellm.drop_params = True`."
+                ),
+                status_code=400,
+            )
+
+    @staticmethod
+    def _model_map_lookup_candidates(model: str) -> List[str]:
+        """Model-map keys to try for ``model``, stripping bedrock/vertex
+        prefixes so a provider-routed Claude still resolves to its entry."""
         candidates = [model]
         for prefix in (
             "bedrock/converse/",
@@ -307,15 +352,40 @@ class AnthropicModelInfo(BaseLLMModelInfo):
                 candidates.append(f"bedrock/{base}")
         except Exception:
             pass
+        return candidates
+
+    @staticmethod
+    def _get_model_capability(model: str, key: str) -> Optional[bool]:
+        """Read boolean capability ``key`` from the model map, or None when
+        no entry declares it."""
         try:
-            for cand in candidates:
-                if cand in litellm.model_cost and (
-                    litellm.model_cost[cand].get(key) is True
-                ):
-                    return True
+            for cand in AnthropicModelInfo._model_map_lookup_candidates(model):
+                value = litellm.model_cost.get(cand, {}).get(key)
+                if isinstance(value, bool):
+                    return value
         except Exception:
             pass
-        return False
+        return None
+
+    @staticmethod
+    def _supports_model_capability(model: str, key: str) -> bool:
+        """Check a boolean capability ``key`` in the model map.
+
+        Strips bedrock/vertex prefixes so a provider-routed Claude still
+        resolves to the Anthropic model-map entry.
+        """
+        from litellm.utils import _supports_factory
+
+        try:
+            if _supports_factory(
+                model=model,
+                custom_llm_provider="anthropic",
+                key=key,
+            ):
+                return True
+        except Exception:
+            pass
+        return AnthropicModelInfo._get_model_capability(model, key) is True
 
     @staticmethod
     def _is_adaptive_thinking_model(model: str) -> bool:

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -1,5 +1,6 @@
 # What is this?
 ## Translates OpenAI call to Anthropic `/v1/messages` format
+import copy
 import json
 import traceback
 from collections import deque
@@ -29,6 +30,98 @@ if TYPE_CHECKING:
     from litellm.types.utils import ModelResponseStream
 
 
+class _CombinedChunkSplitter:
+    """
+    Splits a streaming chunk that carries BOTH response content and a
+    ``finish_reason`` into two chunks: a content-only chunk followed by a
+    finish-only chunk.
+
+    ``AnthropicStreamWrapper`` (via ``translate_streaming_openai_response_to_anthropic``)
+    assumes content and ``finish_reason`` never arrive in the same chunk — true for
+    real provider streams, but false for fake-streamed providers (e.g. Vertex AI
+    Gemma ``:predict``) where ``MockResponseIterator`` collapses the entire response
+    into a single chunk. Without this split the assumption causes all content to be
+    silently dropped (only the ``message_delta`` stop event is emitted).
+
+    Supports both sync and async iteration, since ``AnthropicStreamWrapper`` exposes
+    both ``__next__`` and ``__anext__``. An instance is single-mode: callers must
+    iterate it either synchronously or asynchronously, never both — the two modes
+    hold independent iterator references on the upstream stream and mixing them
+    would advance them out of sync.
+    """
+
+    def __init__(self, completion_stream: Any):
+        self._stream = completion_stream
+        self._sync_iter: Optional[Iterator[Any]] = None
+        self._async_iter: Optional[AsyncIterator[Any]] = None
+        self._buffer: deque = deque()
+
+    @staticmethod
+    def _is_combined(chunk: Any) -> bool:
+        """True if ``chunk`` carries response content AND a finish_reason."""
+        choices = getattr(chunk, "choices", None)
+        if not choices:
+            return False
+        choice = choices[0]
+        if getattr(choice, "finish_reason", None) is None:
+            return False
+        delta = getattr(choice, "delta", None)
+        if delta is None:
+            return False
+        return bool(
+            getattr(delta, "content", None)
+            or getattr(delta, "tool_calls", None)
+            or getattr(delta, "reasoning_content", None)
+            or getattr(delta, "thinking_blocks", None)
+        )
+
+    @staticmethod
+    def _split(chunk: Any) -> List[Any]:
+        """Return ``[chunk]``, or ``[content_chunk, finish_chunk]`` if combined."""
+        if not _CombinedChunkSplitter._is_combined(chunk):
+            return [chunk]
+
+        # Content chunk: keep the delta payload, clear the finish_reason.
+        content_chunk = copy.deepcopy(chunk)
+        content_chunk.choices[0].finish_reason = None
+
+        # Finish chunk: keep finish_reason (and usage), clear the delta payload.
+        finish_chunk = copy.deepcopy(chunk)
+        finish_delta = finish_chunk.choices[0].delta
+        finish_delta.content = None
+        if hasattr(finish_delta, "tool_calls"):
+            finish_delta.tool_calls = None
+        if hasattr(finish_delta, "reasoning_content"):
+            finish_delta.reasoning_content = None
+        if hasattr(finish_delta, "thinking_blocks"):
+            finish_delta.thinking_blocks = None
+        return [content_chunk, finish_chunk]
+
+    def __iter__(self) -> "Iterator[Any]":
+        return self
+
+    def __next__(self) -> Any:
+        if self._buffer:
+            return self._buffer.popleft()
+        if self._sync_iter is None:
+            self._sync_iter = iter(self._stream)
+        chunk = next(self._sync_iter)  # propagates StopIteration when exhausted
+        self._buffer.extend(self._split(chunk))
+        return self._buffer.popleft()
+
+    def __aiter__(self) -> "AsyncIterator[Any]":
+        return self
+
+    async def __anext__(self) -> Any:
+        if self._buffer:
+            return self._buffer.popleft()
+        if self._async_iter is None:
+            self._async_iter = self._stream.__aiter__()
+        chunk = await self._async_iter.__anext__()  # propagates StopAsyncIteration
+        self._buffer.extend(self._split(chunk))
+        return self._buffer.popleft()
+
+
 class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     """
     - first chunk return 'message_start'
@@ -62,7 +155,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         compaction_block: Optional[CompactionBlock] = None,
         iterations_usage: Optional[List[UsageIteration]] = None,
     ):
-        super().__init__(completion_stream)
+        # Wrap the upstream stream so chunks that carry both content and a
+        # finish_reason (fake-streamed providers) are split into two — see
+        # _CombinedChunkSplitter.
+        super().__init__(_CombinedChunkSplitter(completion_stream))
         self.model = model
         # Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
         self.tool_name_mapping = tool_name_mapping or {}

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -155,10 +155,24 @@ class AnthropicResponsesStreamWrapper:
                 event.get("delta", "") if isinstance(event, dict) else ""
             )
             block_idx = (
-                self._item_id_to_block_index.get(item_id, self._current_block_index)
+                self._item_id_to_block_index.get(item_id, -1)
                 if item_id
                 else self._current_block_index
             )
+            if block_idx < 0:
+                # Some providers (e.g. LMStudio) skip response.output_item.added,
+                # so no text block is open yet; synthesize content_block_start
+                # instead of emitting a delta with index -1
+                block_idx = self._next_block_index()
+                if item_id:
+                    self._item_id_to_block_index[item_id] = block_idx
+                self._chunk_queue.append(
+                    {
+                        "type": "content_block_start",
+                        "index": block_idx,
+                        "content_block": {"type": "text", "text": ""},
+                    }
+                )
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",

litellm/llms/base_llm/base_model_iterator.py

@@ -50,6 +50,11 @@ def convert_model_response_to_streaming(
             model=model_response.model,
             choices=streaming_choices,
         )
+        # Carry usage onto the streaming chunk so fake-streamed responses
+        # (e.g. Vertex AI Gemma :predict) still report token counts.
+        usage = getattr(model_response, "usage", None)
+        if usage is not None:
+            setattr(processed_chunk, "usage", usage)
         return processed_chunk
     except Exception as e:
         raise ValueError(

litellm/llms/bedrock/chat/converse_transformation.py

@@ -920,10 +920,15 @@ class AmazonConverseConfig(BaseConfig):
                         continue
                     value = [value]
                 optional_params["stopSequences"] = value
-            if param == "temperature":
-                optional_params["temperature"] = value
-            if param == "top_p":
-                optional_params["topP"] = value
+            if param == "temperature" or param == "top_p":
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=optional_params,
+                    model=model,
+                    param=param,
+                    value=value,
+                    drop_params=drop_params,
+                    output_key="topP" if param == "top_p" else param,
+                )
             if param == "tools" and isinstance(value, list):
                 self._apply_tool_call_transformation(
                     tools=cast(List[OpenAIChatCompletionToolParam], value),
@@ -1221,7 +1226,9 @@ class AmazonConverseConfig(BaseConfig):
             inference_params["topK"] = inference_params.pop("top_k")
         return InferenceConfig(**inference_params)
 
-    def _handle_top_k_value(self, model: str, inference_params: dict) -> dict:
+    def _handle_top_k_value(
+        self, model: str, inference_params: dict, drop_params: bool = False
+    ) -> dict:
         base_model = BedrockModelInfo.get_base_model(model)
 
         val_top_k = None
@@ -1230,16 +1237,25 @@ class AmazonConverseConfig(BaseConfig):
         elif "top_k" in inference_params:
             val_top_k = inference_params.pop("top_k")
 
-        if val_top_k:
+        if val_top_k is not None:
             if base_model.startswith("anthropic"):
-                return {"top_k": val_top_k}
+                top_k_params: dict = {}
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=top_k_params,
+                    model=model,
+                    param="top_k",
+                    value=val_top_k,
+                    drop_params=drop_params,
+                    output_key="top_k",
+                )
+                return top_k_params
             if base_model.startswith("amazon.nova"):
                 return {"inferenceConfig": {"topK": val_top_k}}
 
         return {}
 
     def _prepare_request_params(
-        self, optional_params: dict, model: str
+        self, optional_params: dict, model: str, drop_params: bool = False
     ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]:
         """Prepare and separate request parameters."""
         # Consume the internal ``_output_config_normalized`` marker set by
@@ -1338,7 +1354,7 @@ class AmazonConverseConfig(BaseConfig):
 
         # Only set the topK value in for models that support it
         additional_request_params.update(
-            self._handle_top_k_value(model, inference_params)
+            self._handle_top_k_value(model, inference_params, drop_params)
         )
 
         # Filter out internal/MCP-related parameters that shouldn't be sent to the API
@@ -1572,6 +1588,7 @@ class AmazonConverseConfig(BaseConfig):
         optional_params: dict,
         messages: Optional[List[AllMessageValues]] = None,
         headers: Optional[dict] = None,
+        drop_params: bool = False,
     ) -> CommonRequestObject:
         ## VALIDATE REQUEST
         """
@@ -1618,7 +1635,7 @@ class AmazonConverseConfig(BaseConfig):
             additional_request_params,
             request_metadata,
             output_config,
-        ) = self._prepare_request_params(optional_params, model)
+        ) = self._prepare_request_params(optional_params, model, drop_params)
 
         original_tools = inference_params.pop("tools", [])
 
@@ -1701,6 +1718,7 @@ class AmazonConverseConfig(BaseConfig):
             optional_params=optional_params,
             messages=messages,
             headers=headers,
+            drop_params=litellm_params.get("drop_params") is True,
         )
 
         bedrock_messages = (
@@ -1758,6 +1776,7 @@ class AmazonConverseConfig(BaseConfig):
             optional_params=optional_params,
             messages=messages,
             headers=headers,
+            drop_params=litellm_params.get("drop_params") is True,
         )
 
         ## TRANSFORMATION ##

litellm/llms/bedrock/count_tokens/transformation.py

@@ -11,6 +11,11 @@ from typing import Any, Dict, List, Optional
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 
+# Placeholder satisfying the Anthropic InvokeModel schema's required
+# max_tokens field; CountTokens only counts input, so it has no effect
+# on any generation.
+DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS = 1024
+
 
 class BedrockCountTokensConfig(BaseAWSLLM):
     """
@@ -32,8 +37,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         Returns:
             'converse' or 'invokeModel'
         """
-        # If the request has messages in the expected Anthropic format, use converse
-        if "messages" in request_data and isinstance(request_data["messages"], list):
+        messages = request_data.get("messages")
+        if isinstance(messages, list):
+            # Anthropic content blocks carry a "type" key ({"type": "text", ...});
+            # Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
+            # rejects Anthropic-shape blocks, so route those to invokeModel,
+            # which forwards the body verbatim.
+            for message in messages:
+                if not isinstance(message, dict):
+                    continue
+                content = message.get("content")
+                if isinstance(content, list) and any(
+                    isinstance(block, dict) and "type" in block for block in content
+                ):
+                    return "invokeModel"
             return "converse"
 
         # For raw text or other formats, use invokeModel
@@ -68,7 +85,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         {
             "input": {
                 "invokeModel": {
-                    "body": "{...raw model input...}"
+                    "body": "<base64-encoded raw model input>"
                 }
             }
         }
@@ -168,13 +185,24 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         self, request_data: Dict[str, Any]
     ) -> Dict[str, Any]:
         """Transform to InvokeModel input format."""
+        import base64
         import json
 
         # For InvokeModel, we need to provide the raw body that would be sent to the model
         # Remove the 'model' field from the body as it's not part of the model input
         body_data = {k: v for k, v in request_data.items() if k != "model"}
 
-        return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
+        if "messages" in body_data:
+            # Bedrock validates the body against the model's InvokeModel schema;
+            # Anthropic Messages bodies require these fields.
+            body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
+            body_data.setdefault(
+                "max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
+            )
+
+        # The CountTokens API expects invokeModel.body as a base64-encoded blob
+        encoded_body = base64.b64encode(json.dumps(body_data).encode()).decode()
+        return {"input": {"invokeModel": {"body": encoded_body}}}
 
     def get_bedrock_count_tokens_endpoint(
         self,

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -1,8 +1,10 @@
 """
 Amazon Bedrock Mantle - Responses API backend.
 
-gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses`
-path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI
+Mantle serves Responses on two upstream paths: gpt frontier models (gpt-5.5 /
+gpt-5.4) on `/openai/v1/responses`, and everything else that supports Responses
+(e.g. gpt-oss) on the standard `/v1/responses`. The gate picks the path per
+model and injects it via `use_openai_path`. Payloads and SSE follow the OpenAI
 Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides
 only the endpoint URL and authentication.
 
@@ -48,9 +50,14 @@ _MANTLE_HOST_RE = re.compile(
 
 
 class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
-    def __init__(self, aws_signer: Optional[BaseAWSLLM] = None):
+    def __init__(
+        self,
+        aws_signer: Optional[BaseAWSLLM] = None,
+        use_openai_path: bool = True,
+    ):
         super().__init__()
         self._aws_signer = aws_signer or BaseAWSLLM()
+        self.use_openai_path = use_openai_path
 
     @property
     def custom_llm_provider(self) -> LlmProviders:
@@ -94,7 +101,8 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
         # single resolved region so aws_region_name wins; preserve custom proxy hosts.
         if _MANTLE_HOST_RE.match(base):
             base = f"https://bedrock-mantle.{region}.api.aws"
-        return f"{base}/openai/v1/responses"
+        path = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
+        return f"{base}{path}"
 
     def validate_environment(
         self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]

litellm/llms/databricks/streaming_utils.py

@@ -25,6 +25,28 @@ class ModelResponseIterator:
             finish_reason = ""
             usage: Optional[ChatCompletionUsageBlock] = None
 
+            # Usage-only final chunk (OpenAI ``stream_options.include_usage``)
+            # arrives with an empty ``choices`` list — return usage without
+            # indexing ``choices[0]``.
+            if len(processed_chunk.choices) == 0:
+                final_usage = getattr(processed_chunk, "usage", None)
+                return GenericStreamingChunk(
+                    text="",
+                    tool_use=None,
+                    is_finished=False,
+                    finish_reason="",
+                    usage=(
+                        ChatCompletionUsageBlock(
+                            prompt_tokens=final_usage.prompt_tokens or 0,
+                            completion_tokens=final_usage.completion_tokens or 0,
+                            total_tokens=final_usage.total_tokens or 0,
+                        )
+                        if final_usage is not None
+                        else None
+                    ),
+                    index=0,
+                )
+
             if processed_chunk.choices[0].delta.content is not None:  # type: ignore
                 text = processed_chunk.choices[0].delta.content  # type: ignore
 

litellm/llms/litellm_proxy/skills/README.md

@@ -18,7 +18,7 @@ flowchart TB
         F[Request with container.skills] --> G[SkillsInjectionHook]
         G --> H{skill_id prefix?}
         
-        H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
+        H -->|"litellm_skill_abc"| I[Fetch from LiteLLM DB]
         H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
         
         I --> K{Model provider?}
@@ -57,7 +57,7 @@ sequenceDiagram
     
     Note over LiteLLM,PreHook: PRE-CALL HOOK
     LiteLLM->>PreHook: Intercept request
-    PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
+    PreHook->>PreHook: Fetch skill from DB (litellm_skill_id)
     PreHook->>PreHook: Extract SKILL.md from ZIP
     PreHook->>PreHook: Inject SKILL.md into system prompt
     PreHook->>PreHook: Add litellm_code_execution tool
@@ -105,7 +105,7 @@ response = await litellm.acompletion(
     model="gpt-4o-mini",
     messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
     container={
-        "skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
+        "skills": [{"type": "custom", "skill_id": "litellm_skill_abc123"}]
     },
 )
 
@@ -261,7 +261,7 @@ response = litellm.completion(
     messages=[{"role": "user", "content": "Analyze this data..."}],
     container={
         "skills": [
-            {"type": "custom", "skill_id": "litellm:skill_abc123"}  # litellm: prefix
+            {"type": "custom", "skill_id": "litellm_skill_abc123"}  # litellm_skill_ prefix
         ]
     }
 )
@@ -277,7 +277,7 @@ response = litellm.completion(
     "messages": [{"role": "user", "content": "Help me analyze data"}],
     "container": {
         "skills": [
-            {"type": "custom", "skill_id": "litellm:skill_abc123"}
+            {"type": "custom", "skill_id": "litellm_skill_abc123"}
         ]
     }
 }
@@ -287,7 +287,7 @@ response = litellm.completion(
 
 The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
 
-1. **Detects `litellm:` prefix** → Fetches skill from database
+1. **Detects `litellm_skill_` prefix** → Fetches skill from database
 2. **Checks model provider** → Bedrock is not Anthropic
 3. **Extracts SKILL.md** from stored ZIP file
 4. **Converts skill to tool** + **Injects content into system prompt**
@@ -361,8 +361,8 @@ model LiteLLM_SkillsTable {
 | Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
 | Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
 | Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
-| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
-| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
+| Use LiteLLM skill on Anthropic | N/A | `litellm_skill_abc` | Convert to tools |
+| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm_skill_abc` | Convert to tools + inject SKILL.md |
 
 ## Testing
 

litellm/llms/litellm_proxy/skills/constants.py

@@ -4,6 +4,10 @@ Constants for LiteLLM Skills
 Centralized constants for skills processing, code execution, and sandbox configuration.
 """
 
+LITELLM_SKILL_ID_PREFIX: str = "litellm_skill_"
+"""Prefix for DB-backed skill IDs. The model-facing tool name is the skill ID
+with hyphens/spaces replaced by underscores, which leaves this prefix intact."""
+
 # Code execution loop settings
 DEFAULT_MAX_ITERATIONS: int = 10
 """Maximum number of iterations for the automatic code execution loop."""

litellm/llms/litellm_proxy/skills/handler.py

@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional
 
 from litellm._logging import verbose_logger
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.llms.litellm_proxy.skills.constants import LITELLM_SKILL_ID_PREFIX
 from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest, UserAPIKeyAuth
 from litellm.proxy.common_utils.resource_ownership import (
     get_primary_resource_owner_scope,
@@ -68,7 +69,7 @@ class LiteLLMSkillsHandler:
     ) -> LiteLLM_SkillsTable:
         prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
 
-        skill_id = f"litellm_skill_{uuid.uuid4()}"
+        skill_id = f"{LITELLM_SKILL_ID_PREFIX}{uuid.uuid4()}"
         owner = get_primary_resource_owner_scope(user_api_key_dict) or user_id
         if owner is None:
             # Identity-less callers (no user_id / team_id / org_id /

litellm/llms/openai/completion/handler.py

@@ -49,6 +49,8 @@ class OpenAITextCompletion(BaseLLM):
         headers: Optional[dict] = None,
     ):
         try:
+            if headers:
+                optional_params = {**optional_params, "extra_headers": headers}
             if headers is None:
                 headers = self.validate_environment(api_key=api_key)
             if model is None or messages is None:

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -19,7 +19,7 @@ from litellm.types.llms.vertex_ai import (
     VertexAICachedContentResponseObject,
 )
 
-from ..common_utils import VertexAIError
+from ..common_utils import VertexAIError, get_vertex_base_url
 from ..vertex_llm_base import VertexBase
 from .transformation import (
     separate_cached_messages,
@@ -69,17 +69,13 @@ class ContextCachingEndpoints(VertexBase):
         elif custom_llm_provider == "vertex_ai":
             auth_header = vertex_auth_header
             endpoint = "cachedContents"
-            if vertex_location == "global":
-                url = f"https://aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
-            else:
-                url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
+            base_url = get_vertex_base_url(vertex_location)
+            url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
         else:
             auth_header = vertex_auth_header
             endpoint = "cachedContents"
-            if vertex_location == "global":
-                url = f"https://aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
-            else:
-                url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
+            base_url = get_vertex_base_url(vertex_location)
+            url = f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
 
         return self._check_custom_proxy(
             api_base=api_base,

litellm/llms/xai/chat/transformation.py

@@ -5,6 +5,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
+from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     filter_value_from_dict,
     strip_name_from_messages,
@@ -39,6 +40,72 @@ class XAIChatConfig(OpenAIGPTConfig):
         dynamic_api_key = XAIModelInfo.get_api_key(api_key)
         return api_base, dynamic_api_key
 
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        from litellm.llms.xai.oauth import (
+            XAIOAuthAuthenticator,
+            XAIOAuthError,
+            should_use_xai_oauth,
+        )
+
+        dynamic_api_key = XAIModelInfo.get_api_key(api_key)
+        if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
+            try:
+                headers["Authorization"] = (
+                    f"Bearer {XAIOAuthAuthenticator().get_access_token()}"
+                )
+            except XAIOAuthError as exc:
+                raise AuthenticationError(
+                    model=model,
+                    llm_provider=self.custom_llm_provider or "xai",
+                    message=str(exc),
+                ) from exc
+            if "content-type" not in headers and "Content-Type" not in headers:
+                headers["Content-Type"] = "application/json"
+            return headers
+
+        return super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=dynamic_api_key,
+            api_base=api_base,
+        )
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
+
+        dynamic_api_key = XAIModelInfo.get_api_key(api_key)
+        if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
+            api_base = XAIOAuthAuthenticator().get_api_base()
+
+        return super().get_complete_url(
+            api_base=api_base,
+            api_key=dynamic_api_key,
+            model=model,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            stream=stream,
+        )
+
     def get_supported_openai_params(self, model: str) -> list:
         base_openai_params = [
             "logit_bias",

litellm/llms/xai/oauth.py

@@ -0,0 +1,421 @@
+import base64
+import hashlib
+import json
+import os
+import secrets
+import sys
+import threading
+import time
+import uuid
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict, Optional, Tuple, Union
+from urllib.parse import parse_qs, urlencode, urlparse
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.constants import XAI_API_BASE
+from litellm.llms.custom_httpx.http_handler import HTTPHandler, _get_httpx_client
+from litellm.secret_managers.main import get_secret_str
+
+XAI_OAUTH_ISSUER = "https://auth.x.ai"
+XAI_OAUTH_DISCOVERY_URL = f"{XAI_OAUTH_ISSUER}/.well-known/openid-configuration"
+XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"
+XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access"
+XAI_OAUTH_REDIRECT_HOST = "127.0.0.1"
+XAI_OAUTH_REDIRECT_PORT = 56121
+XAI_OAUTH_REDIRECT_PATH = "/callback"
+XAI_OAUTH_EXPIRY_SKEW_SECONDS = 120
+XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS = 180
+_XAI_OAUTH_REFRESH_LOCK = threading.Lock()
+
+
+class XAIOAuthError(Exception):
+    pass
+
+
+class XAIOAuthLoginRequiredError(XAIOAuthError):
+    pass
+
+
+class _CallbackHandler(BaseHTTPRequestHandler):
+    server: "_CallbackServer"
+
+    def do_GET(self) -> None:
+        parsed = urlparse(self.path)
+        if parsed.path != XAI_OAUTH_REDIRECT_PATH:
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        params = parse_qs(parsed.query)
+        result = {
+            "code": params.get("code", [None])[0],
+            "state": params.get("state", [None])[0],
+            "error": params.get("error", [None])[0],
+            "error_description": params.get("error_description", [None])[0],
+        }
+        self.server.callback_result = result
+
+        if result["state"] != self.server.expected_state:
+            self.send_response(400)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(
+                b"<html><body><h1>xAI authorization state mismatch.</h1></body></html>"
+            )
+            return
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.end_headers()
+        body = (
+            b"<html><body><h1>xAI authorization failed.</h1>You can close this tab.</body></html>"
+            if result["error"]
+            else b"<html><body><h1>xAI authorization received.</h1>You can close this tab.</body></html>"
+        )
+        self.wfile.write(body)
+
+    def log_message(self, format: str, *args: Any) -> None:
+        return
+
+
+class _CallbackServer(HTTPServer):
+    expected_state: str
+    callback_result: Optional[Dict[str, Optional[str]]]
+
+
+class XAIOAuthAuthenticator:
+    def __init__(
+        self, http_client: Optional[Union[httpx.Client, HTTPHandler]] = None
+    ) -> None:
+        self.token_dir = get_secret_str("XAI_OAUTH_TOKEN_DIR") or os.path.expanduser(
+            "~/.config/litellm/xai_oauth"
+        )
+        self.auth_file = os.path.join(
+            self.token_dir, get_secret_str("XAI_OAUTH_AUTH_FILE") or "auth.json"
+        )
+        self.http_client = http_client
+
+    def get_api_base(self) -> str:
+        return (
+            get_secret_str("XAI_OAUTH_API_BASE")
+            or get_secret_str("XAI_API_BASE")
+            or XAI_API_BASE
+        )
+
+    def get_access_token(self) -> str:
+        auth_data = self._read_auth_file()
+        if not auth_data:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth login required. Run `litellm xai-oauth login`."
+            )
+
+        access_token = auth_data.get("access_token")
+        if access_token and not self._is_expired(auth_data):
+            return access_token
+
+        refresh_token = auth_data.get("refresh_token")
+        if not refresh_token:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
+            )
+
+        with _XAI_OAUTH_REFRESH_LOCK:
+            locked_auth_data = self._read_auth_file() or auth_data
+            access_token = locked_auth_data.get("access_token")
+            if access_token and not self._is_expired(locked_auth_data):
+                return access_token
+
+            refreshed = self._refresh_tokens(locked_auth_data)
+            return refreshed["access_token"]
+
+    def login(self, force: bool = False, no_browser: bool = False) -> Dict[str, Any]:
+        existing = self._read_auth_file()
+        if existing and not force and existing.get("access_token"):
+            if not self._is_expired(existing):
+                return existing
+            if existing.get("refresh_token"):
+                try:
+                    return self._refresh_tokens(existing)
+                except XAIOAuthError:
+                    pass
+
+        discovery = self._discover()
+        verifier, challenge = self._pkce_pair()
+        state = uuid.uuid4().hex
+        nonce = uuid.uuid4().hex
+        server, redirect_uri = self._start_callback_server(state)
+        authorize_url = self._build_authorize_url(
+            authorization_endpoint=discovery["authorization_endpoint"],
+            redirect_uri=redirect_uri,
+            challenge=challenge,
+            state=state,
+            nonce=nonce,
+        )
+
+        if no_browser or not webbrowser.open(authorize_url):
+            sys.stdout.write(
+                f"Open this URL to authenticate with xAI:\n{authorize_url}\n"
+            )
+            sys.stdout.flush()
+
+        result = self._wait_for_callback(server)
+        if result.get("state") != state:
+            raise XAIOAuthError("xAI OAuth state mismatch")
+        if result.get("error"):
+            description = result.get("error_description") or result["error"]
+            raise XAIOAuthError(f"xAI authorization failed: {description}")
+        code = result.get("code")
+        if not code:
+            raise XAIOAuthError("xAI authorization failed: no code returned")
+
+        token_payload = self._exchange_token(
+            discovery["token_endpoint"],
+            {
+                "grant_type": "authorization_code",
+                "code": code,
+                "redirect_uri": redirect_uri,
+                "client_id": XAI_OAUTH_CLIENT_ID,
+                "code_verifier": verifier,
+            },
+        )
+        auth_data = self._build_auth_record(token_payload, discovery["token_endpoint"])
+        self._write_auth_file(auth_data)
+        return auth_data
+
+    def _client(self) -> Union[httpx.Client, HTTPHandler]:
+        return self.http_client or _get_httpx_client()
+
+    def _ensure_token_dir(self) -> None:
+        os.makedirs(self.token_dir, mode=0o700, exist_ok=True)
+        try:
+            os.chmod(self.token_dir, 0o700)
+        except OSError:
+            verbose_logger.debug("Could not chmod xAI OAuth token directory")
+
+    def _read_auth_file(self) -> Optional[Dict[str, Any]]:
+        try:
+            with open(self.auth_file, "r") as f:
+                data = json.load(f)
+            return data if isinstance(data, dict) else None
+        except (IOError, json.JSONDecodeError):
+            return None
+
+    def _write_auth_file(self, data: Dict[str, Any]) -> None:
+        self._ensure_token_dir()
+        tmp_file = os.path.join(
+            self.token_dir,
+            f".{os.path.basename(self.auth_file)}.{uuid.uuid4().hex}.tmp",
+        )
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        fd = os.open(tmp_file, flags, 0o600)
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(data, f)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(tmp_file, self.auth_file)
+            try:
+                os.chmod(self.auth_file, 0o600)
+            except OSError:
+                verbose_logger.debug("Could not chmod xAI OAuth auth file")
+        except Exception:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+            try:
+                os.unlink(tmp_file)
+            except OSError:
+                pass
+            raise
+
+    def _is_expired(self, auth_data: Dict[str, Any]) -> bool:
+        expires_at = auth_data.get("expires_at")
+        if expires_at is None:
+            return True
+        try:
+            return time.time() >= float(expires_at) - XAI_OAUTH_EXPIRY_SKEW_SECONDS
+        except (TypeError, ValueError):
+            return True
+
+    def _discover(self) -> Dict[str, str]:
+        try:
+            response = self._client().get(
+                XAI_OAUTH_DISCOVERY_URL, headers={"Accept": "application/json"}
+            )
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise XAIOAuthError(
+                f"xAI OAuth discovery request failed: {exc.response.status_code} {exc.response.text}"
+            ) from exc
+        try:
+            data = response.json()
+        except ValueError as exc:
+            raise XAIOAuthError(
+                "xAI OAuth discovery response was not valid JSON"
+            ) from exc
+        authorization_endpoint = data.get("authorization_endpoint")
+        token_endpoint = data.get("token_endpoint")
+        if not authorization_endpoint or not token_endpoint:
+            raise XAIOAuthError("xAI OAuth discovery missing endpoints")
+        return {
+            "authorization_endpoint": self._validate_xai_endpoint(
+                authorization_endpoint
+            ),
+            "token_endpoint": self._validate_xai_endpoint(token_endpoint),
+        }
+
+    def _validate_xai_endpoint(self, url: str) -> str:
+        parsed = urlparse(url)
+        host = (parsed.hostname or "").lower()
+        if parsed.scheme != "https" or (host != "x.ai" and not host.endswith(".x.ai")):
+            raise XAIOAuthError(
+                f"xAI OAuth discovery returned unexpected endpoint: {url}"
+            )
+        return url
+
+    def _pkce_pair(self) -> Tuple[str, str]:
+        verifier = (
+            base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode()
+        )
+        challenge = (
+            base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
+            .rstrip(b"=")
+            .decode()
+        )
+        return verifier, challenge
+
+    def _start_callback_server(self, state: str) -> Tuple[_CallbackServer, str]:
+        last_error: Optional[OSError] = None
+        for port in (XAI_OAUTH_REDIRECT_PORT, 0):
+            try:
+                server = _CallbackServer(
+                    (XAI_OAUTH_REDIRECT_HOST, port), _CallbackHandler
+                )
+                server.expected_state = state
+                server.callback_result = None
+                actual_port = server.server_address[1]
+                redirect_uri = f"http://{XAI_OAUTH_REDIRECT_HOST}:{actual_port}{XAI_OAUTH_REDIRECT_PATH}"
+                return server, redirect_uri
+            except OSError as exc:
+                last_error = exc
+        raise XAIOAuthError(f"Could not start xAI OAuth callback server: {last_error}")
+
+    def _build_authorize_url(
+        self,
+        authorization_endpoint: str,
+        redirect_uri: str,
+        challenge: str,
+        state: str,
+        nonce: str,
+    ) -> str:
+        params = {
+            "response_type": "code",
+            "client_id": XAI_OAUTH_CLIENT_ID,
+            "redirect_uri": redirect_uri,
+            "scope": XAI_OAUTH_SCOPE,
+            "code_challenge": challenge,
+            "code_challenge_method": "S256",
+            "state": state,
+            "nonce": nonce,
+        }
+        return f"{authorization_endpoint}?{urlencode(params)}"
+
+    def _wait_for_callback(self, server: _CallbackServer) -> Dict[str, Optional[str]]:
+        server.timeout = 1
+        deadline = time.time() + XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS
+        try:
+            while time.time() < deadline:
+                server.handle_request()
+                if server.callback_result is not None:
+                    return server.callback_result
+        finally:
+            server.server_close()
+        raise XAIOAuthError("Timed out waiting for xAI OAuth callback")
+
+    def _exchange_token(
+        self, token_endpoint: str, data: Dict[str, str]
+    ) -> Dict[str, Any]:
+        try:
+            response = self._client().post(
+                token_endpoint,
+                headers={
+                    "Accept": "application/json",
+                    "Content-Type": "application/x-www-form-urlencoded",
+                },
+                data=data,
+            )
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise XAIOAuthError(
+                f"xAI OAuth token request failed: {exc.response.status_code} {exc.response.text}"
+            ) from exc
+        try:
+            body = response.json()
+        except ValueError as exc:
+            raise XAIOAuthError("xAI OAuth token response was not valid JSON") from exc
+        if not isinstance(body, dict):
+            raise XAIOAuthError("xAI OAuth token response was not an object")
+        return body
+
+    def _build_auth_record(
+        self,
+        token_payload: Dict[str, Any],
+        token_endpoint: str,
+        fallback_refresh_token: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        access_token = token_payload.get("access_token")
+        refresh_token = token_payload.get("refresh_token") or fallback_refresh_token
+        if not access_token:
+            raise XAIOAuthError("xAI OAuth token response missing access_token")
+        if not refresh_token:
+            raise XAIOAuthError("xAI OAuth token response missing refresh_token")
+        expires_in = token_payload.get("expires_in") or 3600
+        try:
+            expires_at = int(time.time() + int(expires_in))
+        except (TypeError, ValueError):
+            expires_at = int(time.time() + 3600)
+        return {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "id_token": token_payload.get("id_token"),
+            "token_type": token_payload.get("token_type") or "Bearer",
+            "token_endpoint": token_endpoint,
+            "expires_at": expires_at,
+        }
+
+    def _refresh_tokens(self, auth_data: Dict[str, Any]) -> Dict[str, Any]:
+        token_endpoint = auth_data.get("token_endpoint")
+        if not token_endpoint:
+            token_endpoint = self._discover()["token_endpoint"]
+        token_endpoint = self._validate_xai_endpoint(token_endpoint)
+        refresh_token = auth_data.get("refresh_token")
+        if not refresh_token:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
+            )
+
+        token_payload = self._exchange_token(
+            token_endpoint,
+            {
+                "grant_type": "refresh_token",
+                "refresh_token": refresh_token,
+                "client_id": XAI_OAUTH_CLIENT_ID,
+            },
+        )
+        refreshed = self._build_auth_record(
+            token_payload,
+            token_endpoint,
+            fallback_refresh_token=refresh_token,
+        )
+        self._write_auth_file(refreshed)
+        return refreshed
+
+
+def should_use_xai_oauth(litellm_params: Optional[Dict[str, Any]]) -> bool:
+    return bool((litellm_params or {}).get("use_xai_oauth"))

litellm/llms/xai/responses/transformation.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
+from litellm.exceptions import AuthenticationError
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.llms.xai.common_utils import XAIModelInfo
 from litellm.secret_managers.main import get_secret_str
@@ -220,10 +221,27 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
             litellm_params.api_key, legacy_generic_before_env=True
         )
 
+        if not api_key:
+            from litellm.llms.xai.oauth import (
+                XAIOAuthAuthenticator,
+                XAIOAuthError,
+                should_use_xai_oauth,
+            )
+
+            if should_use_xai_oauth(litellm_params.model_dump()):
+                try:
+                    api_key = XAIOAuthAuthenticator().get_access_token()
+                except XAIOAuthError as exc:
+                    raise AuthenticationError(
+                        model=model,
+                        llm_provider=self.custom_llm_provider.value,
+                        message=str(exc),
+                    ) from exc
+
         if not api_key:
             raise ValueError(
                 "XAI API key is required. Set api_key, litellm.xai_key, "
-                "litellm.api_key, or XAI_API_KEY."
+                "litellm.api_key, XAI_API_KEY, or use_xai_oauth=True."
             )
 
         headers.update(
@@ -244,12 +262,20 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         Returns:
             str: The full URL for the XAI /responses endpoint
         """
-        api_base = (
-            api_base
-            or litellm.api_base
-            or get_secret_str("XAI_API_BASE")
-            or XAI_API_BASE
+        from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
+
+        api_key = XAIModelInfo.get_api_key(
+            litellm_params.get("api_key"), legacy_generic_before_env=True
         )
+        if should_use_xai_oauth(litellm_params) and not api_key:
+            api_base = XAIOAuthAuthenticator().get_api_base()
+        else:
+            api_base = (
+                api_base
+                or litellm.api_base
+                or get_secret_str("XAI_API_BASE")
+                or XAI_API_BASE
+            )
 
         # Remove trailing slashes
         api_base = api_base.rstrip("/")

litellm/main.py

@@ -1638,6 +1638,7 @@ def completion(  # type: ignore # noqa: PLR0915
             litellm_request_debug=kwargs.get("litellm_request_debug", False),
             tpm=kwargs.get("tpm"),
             rpm=kwargs.get("rpm"),
+            use_xai_oauth=kwargs.get("use_xai_oauth", False),
         )
         cast(LiteLLMLoggingObj, logging).update_environment_variables(
             model=model,
@@ -2134,9 +2135,6 @@ def completion(  # type: ignore # noqa: PLR0915
 
             headers = headers or litellm.headers
 
-            if extra_headers is not None:
-                optional_params["extra_headers"] = extra_headers
-
             ## LOAD CONFIG - if set
             config = litellm.OpenAITextCompletionConfig.get_config()
             for k, v in config.items():
@@ -2162,6 +2160,7 @@ def completion(  # type: ignore # noqa: PLR0915
             _response = openai_text_completions.completion(
                 model=model,
                 messages=messages,
+                headers=headers,
                 model_response=model_response,
                 print_verbose=print_verbose,
                 api_key=api_key,

litellm/model_prices_and_context_window_backup.json

@@ -1156,6 +1156,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "azure_ai/claude-fable-5": {
+        "input_cost_per_token": 1e-05,
+        "output_cost_per_token": 5e-05,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10170,6 +10345,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10204,6 +10380,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10214,6 +10391,40 @@
         },
         "supports_output_config": true
     },
+    "claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-opus-4-8": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10238,6 +10449,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -24180,9 +24392,12 @@
         "max_output_tokens": 8192
     },
     "minimax/MiniMax-M3": {
-        "input_cost_per_token": 6e-07,
-        "output_cost_per_token": 2.4e-06,
-        "cache_read_input_token_cost": 1.2e-07,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_above_512k_tokens": 6e-07,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_512k_tokens": 2.4e-06,
+        "cache_read_input_token_cost": 6e-08,
+        "cache_read_input_token_cost_above_512k_tokens": 1.2e-07,
         "litellm_provider": "minimax",
         "mode": "chat",
         "supports_function_calling": true,
@@ -24191,7 +24406,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_vision": true,
-        "max_input_tokens": 512000,
+        "max_input_tokens": 1000000,
         "max_output_tokens": 128000
     },
     "mistral.devstral-2-123b": {
@@ -34004,6 +34219,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34032,6 +34248,67 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5@default": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34061,6 +34338,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34090,6 +34368,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -41370,6 +41649,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41389,6 +41669,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41690,5 +41971,164 @@
             "/v1/audio/transcriptions"
         ],
         "supports_audio_input": true
+    },
+    "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 6e-07,
+        "output_cost_per_token": 3.6e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 4.5e-07,
+        "output_cost_per_token": 1.8e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/Qwen/Qwen3.6-27B-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 3.2e-07,
+        "output_cost_per_token": 3.2e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-06,
+        "output_cost_per_token": 4.4e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 202752,
+        "max_output_tokens": 202752,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/deepseek-ai/DeepSeek-V4-Flash": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-07,
+        "output_cost_per_token": 2.8e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/moonshotai/Kimi-K2.6": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 9.6e-07,
+        "output_cost_per_token": 4e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/MiniMaxAI/MiniMax-M2.5": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 1.2e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 196608,
+        "max_output_tokens": 196608,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/google/gemma-4-31B-it": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-07,
+        "output_cost_per_token": 5.6e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/openai/gpt-oss-120b": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 6e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/openai/gpt-oss-20b": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 7e-08,
+        "output_cost_per_token": 2.8e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
     }
 }

litellm/proxy/_experimental/mcp_server/mcp_context.py

@@ -19,3 +19,9 @@ _mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
 _mcp_gateway_initialize_instructions: ContextVar[Optional[str]] = ContextVar(
     "_mcp_gateway_initialize_instructions", default=None
 )
+
+# Per-request scoped server name; set in MCP HTTP/SSE handlers when the path
+# identifies exactly one upstream server. Never populated from client-supplied headers.
+_mcp_gateway_server_name: ContextVar[Optional[str]] = ContextVar(
+    "_mcp_gateway_server_name", default=None
+)

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -354,6 +354,52 @@ def _deserialize_json_list(data: Any) -> Optional[List[Dict[str, Any]]]:
     ]
 
 
+def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None:
+    """Coerce ``mcp_server_cost_info`` numeric fields to ``float`` at ingest.
+
+    YAML 1.1 parses scientific notation without a decimal point (e.g.
+    ``7e-05``) as a string, and ``MCPServerCostInfo`` is a TypedDict with no
+    runtime validation, so string-typed costs flow through to the UI and
+    crash its ``.toFixed`` formatting. Values that cannot be coerced are
+    dropped with a warning instead of failing the server load.
+    """
+    cost_info = mcp_info.get("mcp_server_cost_info")
+    if not isinstance(cost_info, dict):
+        return
+
+    server_name = mcp_info.get("server_name")
+    normalized = dict(cost_info)
+
+    default_cost = normalized.get("default_cost_per_query")
+    if default_cost is not None:
+        try:
+            normalized["default_cost_per_query"] = float(default_cost)
+        except (TypeError, ValueError):
+            verbose_logger.warning(
+                "MCP server '%s' has non-numeric default_cost_per_query %r; ignoring it",
+                server_name,
+                default_cost,
+            )
+            del normalized["default_cost_per_query"]
+
+    tool_costs = normalized.get("tool_name_to_cost_per_query")
+    if isinstance(tool_costs, dict):
+        normalized_tool_costs = {}
+        for tool_name, cost in tool_costs.items():
+            try:
+                normalized_tool_costs[tool_name] = float(cost)
+            except (TypeError, ValueError):
+                verbose_logger.warning(
+                    "MCP server '%s' has non-numeric cost %r for tool '%s'; ignoring it",
+                    server_name,
+                    cost,
+                    tool_name,
+                )
+        normalized["tool_name_to_cost_per_query"] = normalized_tool_costs
+
+    mcp_info["mcp_server_cost_info"] = normalized
+
+
 def _create_sampling_callback(user_api_key_auth: Optional[Any] = None):
     """
     Create a sampling callback for MCP ClientSession.
@@ -621,6 +667,7 @@ class MCPServerManager:
                 mcp_info["server_name"] = server_name
             if "description" not in mcp_info and server_config.get("description"):
                 mcp_info["description"] = server_config.get("description")
+            _normalize_mcp_server_cost_info(mcp_info)
 
             # Use alias for name if present, else server_name
             alias = server_config.get("alias", None)
@@ -1091,6 +1138,7 @@ class MCPServerManager:
             mcp_info["server_name"] = mcp_server.server_name or mcp_server.server_id
         if "description" not in mcp_info and mcp_server.description:
             mcp_info["description"] = mcp_server.description
+        _normalize_mcp_server_cost_info(mcp_info)
 
         auth_type = cast(MCPAuthType, mcp_server.auth_type)
         server_url = mcp_server.url

litellm/proxy/_experimental/mcp_server/server.py

@@ -47,6 +47,7 @@ from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
 from litellm.proxy._experimental.mcp_server.mcp_context import (
     _mcp_active_toolset_id,
     _mcp_gateway_initialize_instructions,
+    _mcp_gateway_server_name,
 )
 from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
 from litellm.proxy._experimental.mcp_server.utils import (
@@ -323,10 +324,14 @@ if MCP_AVAILABLE:
             notification_options=notification_options,
             experimental_capabilities=experimental_capabilities or {},
         )
+        updates: Dict[str, Any] = {}
         merged = _mcp_gateway_initialize_instructions.get()
         if merged is not None:
-            return opts.model_copy(update={"instructions": merged})
-        return opts
+            updates["instructions"] = merged
+        scoped_server_name = _mcp_gateway_server_name.get()
+        if scoped_server_name is not None:
+            updates["server_name"] = scoped_server_name
+        return opts.model_copy(update=updates) if updates else opts
 
     ########################################################
     ############ Initialize the MCP Server #################
@@ -1544,6 +1549,7 @@ if MCP_AVAILABLE:
         user_api_key_auth: Optional[UserAPIKeyAuth],
         mcp_servers: Optional[List[str]],
         client_ip: Optional[str],
+        scoped_server_endpoint: bool = False,
     ) -> AsyncIterator[None]:
         allowed = await _get_allowed_mcp_servers(
             user_api_key_auth=user_api_key_auth,
@@ -1565,11 +1571,22 @@ if MCP_AVAILABLE:
                 return_exceptions=True,
             )
         merged = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
-        tok = _mcp_gateway_initialize_instructions.set(merged)
+        scoped_server_name = None
+        if scoped_server_endpoint and len(allowed) == 1:
+            scoped_server = allowed[0]
+            scoped_server_name = (
+                scoped_server.alias
+                or scoped_server.server_name
+                or scoped_server.name
+                or scoped_server.server_id
+            )
+        instructions_token = _mcp_gateway_initialize_instructions.set(merged)
+        server_name_token = _mcp_gateway_server_name.set(scoped_server_name)
         try:
             yield
         finally:
-            _mcp_gateway_initialize_instructions.reset(tok)
+            _mcp_gateway_initialize_instructions.reset(instructions_token)
+            _mcp_gateway_server_name.reset(server_name_token)
 
     async def _get_tools_from_mcp_servers(  # noqa: PLR0915
         user_api_key_auth: Optional[UserAPIKeyAuth],
@@ -3620,6 +3637,7 @@ if MCP_AVAILABLE:
                 oauth2_headers,
                 raw_headers,
             ) = await extract_mcp_auth_context(scope, path)
+            scoped_server_endpoint = len(_get_mcp_servers_in_path(path) or []) == 1
 
             # Extract client IP for MCP access control
             _client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope))
@@ -3896,6 +3914,7 @@ if MCP_AVAILABLE:
                     user_api_key_auth,
                     mcp_servers,
                     _client_ip,
+                    scoped_server_endpoint=scoped_server_endpoint,
                 ):
                     await target_manager.handle_request(scope, receive, local_send)
                     if use_stateful and session_id and scope.get("method") == "DELETE":
@@ -3980,6 +3999,7 @@ if MCP_AVAILABLE:
                 oauth2_headers,
                 raw_headers,
             ) = await extract_mcp_auth_context(scope, path)
+            scoped_server_endpoint = len(_get_mcp_servers_in_path(path) or []) == 1
 
             # Extract client IP for MCP access control
             _sse_client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope))
@@ -4052,6 +4072,7 @@ if MCP_AVAILABLE:
                 user_api_key_auth,
                 mcp_servers,
                 _sse_client_ip,
+                scoped_server_endpoint=scoped_server_endpoint,
             ):
                 await sse_session_manager.handle_request(scope, receive, send)
         except MCPUpstreamAuthError as e:

litellm/proxy/_types.py

@@ -23,6 +23,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
 from litellm.types.integrations.slack_alerting import AlertType
 from litellm.types.llms.openai import (
     AllMessageValues,
+    ResponsesAPIResponse,
 )
 from litellm.types.mcp import (
     MCPAuthType,
@@ -3834,6 +3835,7 @@ PassThroughEndpointLoggingResultValues = Union[
     EmbeddingResponse,
     VideoObject,
     StandardPassThroughResponseObject,
+    ResponsesAPIResponse,
 ]
 
 

litellm/proxy/auth/auth_checks.py

@@ -3516,10 +3516,13 @@ async def _virtual_key_max_budget_check(
     if valid_token.max_budget is not None:
         from litellm.proxy.proxy_server import get_current_spend
 
+        fallback_spend = valid_token.spend or 0.0
+        counter_key = f"spend:key:{valid_token.token}"
+
         # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
         spend = await get_current_spend(
-            counter_key=f"spend:key:{valid_token.token}",
-            fallback_spend=valid_token.spend or 0.0,
+            counter_key=counter_key,
+            fallback_spend=fallback_spend,
         )
 
         ####################################

litellm/proxy/client/README.md

@@ -338,9 +338,9 @@ sequenceDiagram
 
 The CLI provides three authentication commands:
 
-- **`litellm-proxy login`** - Start SSO authentication flow
-- **`litellm-proxy logout`** - Clear stored authentication token
-- **`litellm-proxy whoami`** - Show current authentication status
+- **`lite login`** - Start SSO authentication flow
+- **`lite logout`** - Clear stored authentication token
+- **`lite whoami`** - Show current authentication status
 
 ### Authentication Flow Steps
 
@@ -382,14 +382,14 @@ Once authenticated, the CLI will automatically use the stored token for all requ
 
 ```bash
 # Login
-litellm-proxy login
+lite login
 
 # Use CLI without specifying API key
-litellm-proxy models list
+lite models list
 
 # Check authentication status
-litellm-proxy whoami
+lite whoami
 
 # Logout
-litellm-proxy logout
+lite logout
 ``` 

litellm/proxy/client/cli/README.md

@@ -22,11 +22,11 @@ The CLI can be configured using environment variables or command-line options:
 Example:
 
 ```bash
-litellm-proxy version
+lite version
 # or
-litellm-proxy --version
+lite --version
 # or
-litellm-proxy -v
+lite -v
 ```
 
 ## Commands
@@ -40,7 +40,7 @@ The CLI provides several commands for managing models on your LiteLLM proxy serv
 View all available models:
 
 ```bash
-litellm-proxy models list [--format table|json]
+lite models list [--format table|json]
 ```
 
 Options:
@@ -52,7 +52,7 @@ Options:
 Get detailed information about all models:
 
 ```bash
-litellm-proxy models info [options]
+lite models info [options]
 ```
 
 Options:
@@ -75,7 +75,7 @@ Default columns: `public_model`, `upstream_model`, `updated_at`
 Add a new model to the proxy:
 
 ```bash
-litellm-proxy models add <model-name> [options]
+lite models add <model-name> [options]
 ```
 
 Options:
@@ -86,7 +86,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.com -i description="GPT-4 model"
+lite models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.com -i description="GPT-4 model"
 ```
 
 #### Get Model Info
@@ -94,7 +94,7 @@ litellm-proxy models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.
 Get information about a specific model:
 
 ```bash
-litellm-proxy models get [--id MODEL_ID] [--name MODEL_NAME]
+lite models get [--id MODEL_ID] [--name MODEL_NAME]
 ```
 
 Options:
@@ -107,7 +107,7 @@ Options:
 Delete a model from the proxy:
 
 ```bash
-litellm-proxy models delete <model-id>
+lite models delete <model-id>
 ```
 
 #### Update Model
@@ -115,7 +115,7 @@ litellm-proxy models delete <model-id>
 Update an existing model's configuration:
 
 ```bash
-litellm-proxy models update <model-id> [options]
+lite models update <model-id> [options]
 ```
 
 Options:
@@ -128,7 +128,7 @@ Options:
 Import models from a YAML file:
 
 ```bash
-litellm-proxy models import models.yaml
+lite models import models.yaml
 ```
 
 Options:
@@ -142,31 +142,31 @@ Examples:
 1. Import all models from a YAML file:
 
 ```bash
-litellm-proxy models import models.yaml
+lite models import models.yaml
 ```
 
 2. Dry run (show what would be imported):
 
 ```bash
-litellm-proxy models import models.yaml --dry-run
+lite models import models.yaml --dry-run
 ```
 
 3. Only import models where the model name contains 'gpt':
 
 ```bash
-litellm-proxy models import models.yaml --only-models-matching-regex gpt
+lite models import models.yaml --only-models-matching-regex gpt
 ```
 
 4. Only import models with access group containing 'beta':
 
 ```bash
-litellm-proxy models import models.yaml --only-access-groups-matching-regex beta
+lite models import models.yaml --only-access-groups-matching-regex beta
 ```
 
 5. Combine both filters:
 
 ```bash
-litellm-proxy models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
+lite models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
 ```
 
 ### Credentials Management
@@ -178,7 +178,7 @@ The CLI provides commands for managing credentials on your LiteLLM proxy server:
 View all available credentials:
 
 ```bash
-litellm-proxy credentials list [--format table|json]
+lite credentials list [--format table|json]
 ```
 
 Options:
@@ -194,7 +194,7 @@ The table format displays:
 Create a new credential:
 
 ```bash
-litellm-proxy credentials create <credential-name> --info <json-string> --values <json-string>
+lite credentials create <credential-name> --info <json-string> --values <json-string>
 ```
 
 Options:
@@ -205,7 +205,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy credentials create azure-cred \
+lite credentials create azure-cred \
   --info '{"custom_llm_provider": "azure"}' \
   --values '{"api_key": "sk-123", "api_base": "https://example.azure.openai.com"}'
 ```
@@ -215,7 +215,7 @@ litellm-proxy credentials create azure-cred \
 Get information about a specific credential:
 
 ```bash
-litellm-proxy credentials get <credential-name>
+lite credentials get <credential-name>
 ```
 
 #### Delete Credential
@@ -223,7 +223,7 @@ litellm-proxy credentials get <credential-name>
 Delete a credential:
 
 ```bash
-litellm-proxy credentials delete <credential-name>
+lite credentials delete <credential-name>
 ```
 
 ### Keys Management
@@ -235,7 +235,7 @@ The CLI provides commands for managing API keys on your LiteLLM proxy server:
 View all API keys:
 
 ```bash
-litellm-proxy keys list [--format table|json] [options]
+lite keys list [--format table|json] [options]
 ```
 
 Options:
@@ -256,7 +256,7 @@ Options:
 Generate a new API key:
 
 ```bash
-litellm-proxy keys generate [options]
+lite keys generate [options]
 ```
 
 Options:
@@ -274,7 +274,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration 24h --key-alias my-key --team-id team123
+lite keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration 24h --key-alias my-key --team-id team123
 ```
 
 #### Delete Keys
@@ -282,7 +282,7 @@ litellm-proxy keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration
 Delete API keys by key or alias:
 
 ```bash
-litellm-proxy keys delete [--keys <comma-separated-keys>] [--key-aliases <comma-separated-aliases>]
+lite keys delete [--keys <comma-separated-keys>] [--key-aliases <comma-separated-aliases>]
 ```
 
 Options:
@@ -293,7 +293,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
+lite keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
 ```
 
 #### Get Key Info
@@ -301,7 +301,7 @@ litellm-proxy keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
 Get information about a specific API key:
 
 ```bash
-litellm-proxy keys info --key <key-hash>
+lite keys info --key <key-hash>
 ```
 
 Options:
@@ -311,7 +311,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys info --key sk-key1
+lite keys info --key sk-key1
 ```
 
 ### User Management
@@ -323,7 +323,7 @@ The CLI provides commands for managing users on your LiteLLM proxy server:
 View all users:
 
 ```bash
-litellm-proxy users list
+lite users list
 ```
 
 #### Get User Info
@@ -331,7 +331,7 @@ litellm-proxy users list
 Get information about a specific user:
 
 ```bash
-litellm-proxy users get --id <user-id>
+lite users get --id <user-id>
 ```
 
 #### Create User
@@ -339,7 +339,7 @@ litellm-proxy users get --id <user-id>
 Create a new user:
 
 ```bash
-litellm-proxy users create --email user@example.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
+lite users create --email user@example.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
 ```
 
 #### Delete User
@@ -347,7 +347,7 @@ litellm-proxy users create --email user@example.com --role internal_user --alias
 Delete one or more users by user_id:
 
 ```bash
-litellm-proxy users delete <user-id-1> <user-id-2>
+lite users delete <user-id-1> <user-id-2>
 ```
 
 ### Chat Commands
@@ -359,7 +359,7 @@ The CLI provides commands for interacting with chat models through your LiteLLM
 Create a chat completion:
 
 ```bash
-litellm-proxy chat completions <model> [options]
+lite chat completions <model> [options]
 ```
 
 Arguments:
@@ -379,12 +379,12 @@ Examples:
 
 1. Simple completion:
 ```bash
-litellm-proxy chat completions gpt-4 -m "user:Hello, how are you?"
+lite chat completions gpt-4 -m "user:Hello, how are you?"
 ```
 
 2. Multi-message conversation:
 ```bash
-litellm-proxy chat completions gpt-4 \
+lite chat completions gpt-4 \
   -m "system:You are a helpful assistant" \
   -m "user:What's the capital of France?" \
   -m "assistant:The capital of France is Paris." \
@@ -393,7 +393,7 @@ litellm-proxy chat completions gpt-4 \
 
 3. With generation parameters:
 ```bash
-litellm-proxy chat completions gpt-4 \
+lite chat completions gpt-4 \
   -m "user:Write a story" \
   --temperature 0.7 \
   --max-tokens 500 \
@@ -409,7 +409,7 @@ The CLI provides commands for making direct HTTP requests to your LiteLLM proxy
 Make an HTTP request to any endpoint:
 
 ```bash
-litellm-proxy http request <method> <uri> [options]
+lite http request <method> <uri> [options]
 ```
 
 Arguments:
@@ -425,19 +425,46 @@ Examples:
 
 1. List models:
 ```bash
-litellm-proxy http request GET /models
+lite http request GET /models
 ```
 
 2. Create a chat completion:
 ```bash
-litellm-proxy http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
+lite http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
 ```
 
 3. Test connection with custom headers:
 ```bash
-litellm-proxy http request GET /health/test_connection -H "X-Custom-Header:value"
+lite http request GET /health/test_connection -H "X-Custom-Header:value"
 ```
 
+### Run a Coding Agent
+
+Launch a coding agent with all of its LLM traffic routed through your LiteLLM proxy. Each supported agent is its own command, so there is nothing to remember beyond the agent's name:
+
+```bash
+lite claude
+lite codex
+lite opencode
+```
+
+Anything you type after the agent name is forwarded to it untouched, so the usual flags keep working:
+
+```bash
+lite claude --resume
+lite codex exec "summarize the repo"
+```
+
+Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
+
+The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol).
+
+Options (these belong to the wrapper, so put them before the agent's own flags):
+
+- `--skip-verify`: Skip the pre-launch key check (useful offline or with non-standard auth).
+
+To pin the model, pass the agent's own model flag (for example `lite claude --model my-proxy-model` or `lite codex -m my-proxy-model`), or export the variable the agent reads (`ANTHROPIC_MODEL` / `ANTHROPIC_SMALL_FAST_MODEL` for Claude Code); the wrapper preserves anything you already have set. Whatever model the agent ends up requesting must exist on the proxy, since requests land on the proxy's `/v1/messages` (Anthropic) or `/v1/chat/completions` and `/v1/responses` (OpenAI) endpoints.
+
 ## Environment Variables
 
 The CLI respects the following environment variables:
@@ -450,37 +477,37 @@ The CLI respects the following environment variables:
 1. List all models in table format:
 
 ```bash
-litellm-proxy models list
+lite models list
 ```
 
 2. Add a new model with parameters:
 
 ```bash
-litellm-proxy models add gpt-4 -p api_key=sk-123 -p max_tokens=2048
+lite models add gpt-4 -p api_key=sk-123 -p max_tokens=2048
 ```
 
 3. Get model information in JSON format:
 
 ```bash
-litellm-proxy models info --format json
+lite models info --format json
 ```
 
 4. Update model parameters:
 
 ```bash
-litellm-proxy models update model-123 -p temperature=0.7 -i description="Updated model"
+lite models update model-123 -p temperature=0.7 -i description="Updated model"
 ```
 
 5. List all credentials in table format:
 
 ```bash
-litellm-proxy credentials list
+lite credentials list
 ```
 
 6. Create a new credential for Azure:
 
 ```bash
-litellm-proxy credentials create azure-prod \
+lite credentials create azure-prod \
   --info '{"custom_llm_provider": "azure"}' \
   --values '{"api_key": "sk-123", "api_base": "https://prod.azure.openai.com"}'
 ```
@@ -488,7 +515,7 @@ litellm-proxy credentials create azure-prod \
 7. Make a custom HTTP request:
 
 ```bash
-litellm-proxy http request POST /chat/completions \
+lite http request POST /chat/completions \
   -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' \
   -H "X-Custom-Header:value"
 ```
@@ -497,29 +524,29 @@ litellm-proxy http request POST /chat/completions \
 
 ```bash
 # List users
-litellm-proxy users list
+lite users list
 
 # Get user info
-litellm-proxy users get --id u1
+lite users get --id u1
 
 # Create a user
-litellm-proxy users create --email a@b.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
+lite users create --email a@b.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
 
 # Delete users
-litellm-proxy users delete u1 u2
+lite users delete u1 u2
 ```
 
 9. Import models from a YAML file (with filters):
 
 ```bash
 # Only import models where the model name contains 'gpt'
-litellm-proxy models import models.yaml --only-models-matching-regex gpt
+lite models import models.yaml --only-models-matching-regex gpt
 
 # Only import models with access group containing 'beta'
-litellm-proxy models import models.yaml --only-access-groups-matching-regex beta
+lite models import models.yaml --only-access-groups-matching-regex beta
 
 # Combine both filters
-litellm-proxy models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
+lite models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
 ```
 
 ## Error Handling

litellm/proxy/client/cli/commands/agents.py

@@ -0,0 +1,303 @@
+import os
+import shutil
+import sys
+from typing import Callable, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple
+
+import click
+import requests
+
+from .auth import get_stored_api_key, login
+
+ANTHROPIC_BASE_URL_ENV = "ANTHROPIC_BASE_URL"
+ANTHROPIC_AUTH_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN"
+ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY"
+OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL"
+OPENAI_API_KEY_ENV = "OPENAI_API_KEY"
+
+PROFILE_ANTHROPIC = "anthropic"
+PROFILE_OPENAI = "openai"
+
+_KNOWN_AGENTS: Dict[str, Tuple[str, FrozenSet[str]]] = {
+    "claude": ("Claude Code", frozenset({PROFILE_ANTHROPIC})),
+    "codex": ("Codex", frozenset({PROFILE_OPENAI})),
+    "opencode": ("OpenCode", frozenset({PROFILE_OPENAI})),
+}
+
+_INSTALL_DOCS: Dict[str, str] = {
+    "claude": "https://docs.claude.com/en/docs/claude-code/setup",
+    "codex": "https://developers.openai.com/codex/cli",
+    "opencode": "https://opencode.ai/docs",
+}
+
+CODEX_PROXY_PROVIDER = "litellm"
+
+
+class AgentRunError(Exception):
+    """Raised for any user-actionable failure while preparing to run an agent."""
+
+
+def agent_profile(command: str) -> Tuple[str, FrozenSet[str]]:
+    """Return the (display name, env profiles) for a wrapped command.
+
+    Known agents map to the API family they speak. Anything else gets both
+    families so it works regardless of which env vars the tool reads.
+    """
+    base = os.path.basename(command)
+    if base in _KNOWN_AGENTS:
+        return _KNOWN_AGENTS[base]
+    return base, frozenset({PROFILE_ANTHROPIC, PROFILE_OPENAI})
+
+
+def build_agent_env(
+    base_env: Mapping[str, str],
+    base_url: str,
+    api_key: str,
+    profiles: FrozenSet[str],
+) -> Dict[str, str]:
+    """Return a copy of base_env wired to route the agent through the proxy.
+
+    Anthropic clients (Claude Code) append /v1/messages to ANTHROPIC_BASE_URL,
+    so it stays the bare proxy root; OpenAI clients (Codex, OpenCode) expect the
+    /v1 suffix on OPENAI_BASE_URL. ANTHROPIC_API_KEY is dropped so a stray
+    Anthropic key cannot win over the bearer token we set.
+    """
+    env = dict(base_env)
+    root = base_url.rstrip("/")
+    if PROFILE_ANTHROPIC in profiles:
+        env[ANTHROPIC_BASE_URL_ENV] = root
+        env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key
+        env.pop(ANTHROPIC_API_KEY_ENV, None)
+    if PROFILE_OPENAI in profiles:
+        env[OPENAI_BASE_URL_ENV] = root + "/v1"
+        env[OPENAI_API_KEY_ENV] = api_key
+    return env
+
+
+def _codex_proxy_args(base_url: str) -> List[str]:
+    """Codex `-c` overrides that point it at the proxy.
+
+    Codex ignores OPENAI_BASE_URL (it always dials api.openai.com), so the env
+    profile alone cannot route it. It does honor a custom provider, so define one
+    inline; supports_websockets=false forces the HTTP/SSE Responses transport
+    because the proxy does not speak the Responses WebSocket protocol. The key is
+    read from OPENAI_API_KEY, which build_agent_env already exports.
+    """
+    root = base_url.rstrip("/") + "/v1"
+    provider = f"model_providers.{CODEX_PROXY_PROVIDER}"
+    return [
+        "-c",
+        f'model_provider="{CODEX_PROXY_PROVIDER}"',
+        "-c",
+        f'{provider}.name="LiteLLM proxy"',
+        "-c",
+        f'{provider}.base_url="{root}"',
+        "-c",
+        f'{provider}.env_key="{OPENAI_API_KEY_ENV}"',
+        "-c",
+        f'{provider}.wire_api="responses"',
+        "-c",
+        f"{provider}.supports_websockets=false",
+    ]
+
+
+_PROXY_ARGS: Dict[str, Callable[[str], List[str]]] = {
+    "codex": _codex_proxy_args,
+}
+
+
+def agent_launch_args(command: str, base_url: str) -> List[str]:
+    """Extra CLI args an agent needs to actually honor the proxy.
+
+    Claude Code and OpenCode respect the exported env vars, so they get nothing
+    here; Codex needs its provider pointed via config overrides.
+    """
+    builder = _PROXY_ARGS.get(os.path.basename(command))
+    return builder(base_url) if builder else []
+
+
+def verify_proxy_key(
+    base_url: str,
+    api_key: str,
+    *,
+    get: Callable[..., requests.Response] = requests.get,
+) -> None:
+    """Probe the proxy with the key so bad creds fail here, not inside the agent.
+
+    Raises AgentRunError when the proxy is unreachable or rejects the key. Other
+    non-2xx responses are tolerated; the agent's own call is the real test.
+    """
+    url = base_url.rstrip("/") + "/v1/models"
+    try:
+        resp = get(url, headers={"Authorization": f"Bearer {api_key}"}, timeout=10)
+    except requests.RequestException as e:
+        raise AgentRunError(
+            f"Could not reach the LiteLLM proxy at {base_url.rstrip('/')}: {e}. "
+            "Is it running, and is --base-url (or LITELLM_PROXY_URL) correct?"
+        )
+    if resp.status_code in (401, 403):
+        raise AgentRunError(
+            f"LiteLLM rejected your key (HTTP {resp.status_code}). "
+            "Run `lite login` to refresh it, or pass a valid --api-key."
+        )
+
+
+def _exec(path: str, args: Sequence[str], env: Mapping[str, str]) -> None:
+    os.execvpe(path, list(args), dict(env))
+
+
+def _restore_controlling_terminal() -> None:
+    """Reattach the controlling terminal to stdin before handing off to the agent.
+
+    Completing the browser SSO login can leave stdin detached from the terminal,
+    which makes a TUI agent like Claude Code start in non-interactive mode and
+    exit immediately. Reopening /dev/tty onto fd 0 gives the agent a live
+    terminal; when stdin is still a tty (no login happened) this is a no-op.
+    """
+    if sys.stdin.isatty():
+        return
+    try:
+        fd = os.open("/dev/tty", os.O_RDONLY)
+    except OSError:
+        return
+    try:
+        os.dup2(fd, 0)
+    finally:
+        os.close(fd)
+
+
+def run_agent(
+    base_url: str,
+    api_key: str,
+    command: Sequence[str],
+    *,
+    skip_verify: bool = False,
+    base_env: Optional[Mapping[str, str]] = None,
+    which: Callable[[str], Optional[str]] = shutil.which,
+    verify: Callable[[str, str], None] = verify_proxy_key,
+    launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _exec,
+    reattach_terminal: Optional[Callable[[], None]] = None,
+) -> None:
+    """Validate, wire the environment, and hand off to the agent.
+
+    On success this replaces the current process and never returns. Raises
+    AgentRunError for missing binaries, an unreachable proxy, or a rejected key.
+    reattach_terminal, when given, runs just before handoff to restore stdin.
+    """
+    if not command:
+        raise AgentRunError("Nothing to run.")
+
+    _, profiles = agent_profile(command[0])
+    binary = which(command[0])
+    if binary is None:
+        docs = _INSTALL_DOCS.get(os.path.basename(command[0]))
+        hint = f" Install it first: {docs}" if docs else ""
+        raise AgentRunError(f"Could not find `{command[0]}` on your PATH.{hint}")
+
+    if not skip_verify:
+        verify(base_url, api_key)
+
+    env = build_agent_env(
+        base_env if base_env is not None else os.environ,
+        base_url,
+        api_key,
+        profiles,
+    )
+    extra_args = agent_launch_args(command[0], base_url)
+    if reattach_terminal is not None:
+        reattach_terminal()
+    launcher(binary, [command[0], *extra_args, *command[1:]], env)
+
+
+def _is_interactive() -> bool:
+    return sys.stdin.isatty()
+
+
+def _resolve_api_key(ctx: click.Context) -> str:
+    base_url = ctx.obj["base_url"]
+    api_key = ctx.obj.get("api_key")
+    if api_key:
+        return api_key
+
+    if not _is_interactive():
+        raise click.ClickException(
+            "No LiteLLM key found. Set LITELLM_PROXY_API_KEY (or pass --api-key) for "
+            "non-interactive use, or run `lite login` from a terminal."
+        )
+
+    click.echo("No LiteLLM credentials found; starting login...")
+    ctx.invoke(login)
+    api_key = get_stored_api_key(expected_base_url=base_url)
+    if not api_key:
+        raise click.ClickException(
+            "Login did not produce an API key; cannot start the agent."
+        )
+    return api_key
+
+
+_SKIP_VERIFY_HELP = "Skip the pre-launch key check against the proxy."
+
+
+def _launch(
+    ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool
+) -> None:
+    base_url = ctx.obj["base_url"]
+    started_interactive = _is_interactive()
+    api_key = _resolve_api_key(ctx)
+
+    display_name, _ = agent_profile(binary)
+    click.echo(
+        f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}"
+    )
+
+    try:
+        run_agent(
+            base_url,
+            api_key,
+            [binary, *args],
+            skip_verify=skip_verify,
+            reattach_terminal=(
+                _restore_controlling_terminal if started_interactive else None
+            ),
+        )
+    except AgentRunError as e:
+        raise click.ClickException(str(e))
+
+
+def _make_agent_command(binary: str, display_name: str) -> click.Command:
+    @click.command(
+        name=binary,
+        context_settings={"ignore_unknown_options": True},
+        short_help=f"Run {display_name} through your LiteLLM proxy",
+    )
+    @click.option("--skip-verify", is_flag=True, default=False, help=_SKIP_VERIFY_HELP)
+    @click.argument("args", nargs=-1, type=click.UNPROCESSED)
+    @click.pass_context
+    def _command(ctx: click.Context, skip_verify: bool, args: Sequence[str]) -> None:
+        _launch(ctx, binary, list(args), skip_verify=skip_verify)
+
+    _command.help = (
+        f"Run {display_name} routed through your LiteLLM proxy.\n\n"
+        f"Logs in with LiteLLM if needed, verifies your key against the proxy, "
+        f"exports the env vars {binary} reads, then hands off. Any arguments are "
+        f"forwarded to `{binary}`."
+    )
+    return _command
+
+
+def agent_commands() -> List[click.Command]:
+    """Build one top-level command per known agent, e.g. `lite claude`."""
+    return [
+        _make_agent_command(binary, name)
+        for binary, (name, _profiles) in _KNOWN_AGENTS.items()
+    ]
+
+
+__all__ = [
+    "agent_commands",
+    "run_agent",
+    "build_agent_env",
+    "agent_launch_args",
+    "verify_proxy_key",
+    "agent_profile",
+    "AgentRunError",
+]

litellm/proxy/client/cli/commands/auth.py

@@ -624,7 +624,7 @@ def whoami():
     token_data = load_token()
 
     if not token_data:
-        click.echo("❌ Not authenticated. Run 'litellm-proxy login' to authenticate.")
+        click.echo("❌ Not authenticated. Run 'lite login' to authenticate.")
         return
 
     click.echo("✅ Authenticated")

litellm/proxy/client/cli/commands/chat.py

@@ -122,13 +122,13 @@ def chat(
     Examples:
 
         # Chat with a specific model
-        litellm-proxy chat gpt-4
+        lite chat gpt-4
 
         # Chat without specifying model (will show model selection)
-        litellm-proxy chat
+        lite chat
 
         # Chat with custom settings
-        litellm-proxy chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
+        lite chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
     """
     console = Console()
 

litellm/proxy/client/cli/interface.py

@@ -80,6 +80,8 @@ def styled_prompt():
 
 def show_commands():
     """Display available commands."""
+    from .commands.agents import agent_commands
+
     commands = [
         ("login", "Authenticate with the LiteLLM proxy server"),
         ("logout", "Clear stored authentication"),
@@ -91,6 +93,9 @@ def show_commands():
         ("keys", "Manage API keys"),
         ("teams", "Manage teams and team assignments"),
         ("users", "Manage users"),
+    ]
+    commands += [(c.name, c.get_short_help_str()) for c in agent_commands()]
+    commands += [
         ("version", "Show version information"),
         ("help", "Show this help message"),
         ("quit", "Exit the interactive session"),
@@ -156,7 +161,7 @@ def execute_command(user_input: str, ctx: click.Context):
     # Execute the command
     try:
         # Create a new argument list for click to parse
-        sys.argv = ["litellm-proxy"] + [command] + args
+        sys.argv = ["lite"] + [command] + args
 
         # Get the command object and invoke it
         cmd = cli.commands[command]

litellm/proxy/client/cli/main.py

@@ -7,6 +7,7 @@ import click
 from litellm._version import version as litellm_version
 from litellm.proxy.client.health import HealthManagementClient
 
+from .commands.agents import agent_commands
 from .commands.auth import get_stored_api_key, login, logout, whoami
 from .commands.chat import chat
 from .commands.credentials import credentials
@@ -112,6 +113,9 @@ cli.add_command(keys)
 cli.add_command(teams)
 # Add the users command group
 cli.add_command(users)
+# Add a top-level command per coding agent (claude, codex, opencode, ...)
+for agent_command in agent_commands():
+    cli.add_command(agent_command)
 
 
 if __name__ == "__main__":

litellm/proxy/common_request_processing.py

@@ -1949,12 +1949,26 @@ class ProxyBaseLLMRequestProcessing:
                 code=status.HTTP_400_BAD_REQUEST,
                 headers=headers,
             )
+        # Extract status_code from the exception if it carries one.
+        # Provider exceptions (NotFoundError, BadRequestError, GeminiError,
+        # VertexAIError, etc.) all have a status_code attribute reflecting
+        # the upstream API response. Use it to return the correct HTTP code
+        # instead of defaulting to 500.
+        _exc_status_code = getattr(e, "status_code", None)
+        if (
+            _exc_status_code is not None
+            and isinstance(_exc_status_code, int)
+            and 400 <= _exc_status_code <= 599
+        ):
+            _code = _exc_status_code
+        else:
+            _code = status.HTTP_500_INTERNAL_SERVER_ERROR
         raise ProxyException(
             message=getattr(e, "message", error_msg),
             type=getattr(e, "type", "None"),
             param=getattr(e, "param", "None"),
             openai_code=getattr(e, "code", None),
-            code=getattr(e, "status_code", 500),
+            code=_code,
             provider_specific_fields=getattr(e, "provider_specific_fields", None),
             headers=headers,
         )

litellm/proxy/common_utils/html_forms/cli_sso_success.py

@@ -135,7 +135,7 @@ def render_cli_sso_success_page() -> str:
                 font-size: 14px;
             }}
 
-            .countdown {{
+            .status {{
                 color: #64748b;
                 font-size: 14px;
                 font-weight: 500;
@@ -183,23 +183,11 @@ def render_cli_sso_success_page() -> str:
                 <p>You can now use LiteLLM CLI commands with your authenticated session.</p>
             </div>
             
-            <div class="countdown" id="countdown">This window will close in 3 seconds...</div>
+            <div class="status">You can now close this window and return to your terminal.</div>
         </div>
-        
+
         <script>
-            let seconds = 3;
-            const countdownElement = document.getElementById('countdown');
-            
-            const countdown = setInterval(function() {{
-                seconds--;
-                if (seconds > 0) {{
-                    countdownElement.textContent = `This window will close in ${{seconds}} second${{seconds === 1 ? '' : 's'}}...`;
-                }} else {{
-                    countdownElement.textContent = 'Closing...';
-                    clearInterval(countdown);
-                    window.close();
-                }}
-            }}, 1000);
+            window.close();
         </script>
     </body>
     </html>

litellm/proxy/db/tool_registry_writer.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ToolDiscoveryQueueItem
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.repositories.table_repositories import ToolRepository
 from litellm.types.tool_management import (
@@ -309,7 +310,11 @@ class ToolPolicyRegistry:
     async def sync_tool_policy_from_db(self, prisma_client: "PrismaClient") -> None:
         """Load all tool policies and object-permission blocked_tools from DB."""
         try:
-            tools = await ToolRepository(prisma_client).table.find_many()
+            tools = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ToolRepository(prisma_client).table.find_many(),
+                reason="sync_tool_policy_from_db_tools_lookup_failure",
+            )
             self._tool_input_policies = {
                 row.tool_name: getattr(row, "input_policy", "untrusted") or "untrusted"
                 for row in tools
@@ -319,7 +324,11 @@ class ToolPolicyRegistry:
                 for row in tools
             }
 
-            perms = await ObjectPermissionRepository(prisma_client).table.find_many()
+            perms = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ObjectPermissionRepository(prisma_client).table.find_many(),
+                reason="sync_tool_policy_from_db_perms_lookup_failure",
+            )
             self._blocked_tools_by_op_id = {}
             for row in perms:
                 op_id = getattr(row, "object_permission_id", None)

litellm/proxy/guardrails/guardrail_hooks/presidio.py

@@ -1225,6 +1225,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
             for chunk in all_chunks:
                 yield chunk
 
+    @staticmethod
+    def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: Dict[str, str]) -> bytes:
+        try:
+            text = chunk.decode("utf-8")
+        except UnicodeDecodeError:
+            return chunk
+
+        result_lines: List[str] = []
+        for line in text.split("\n"):
+            line = line.rstrip("\r")
+            if line.startswith("data: ") and line != "data: [DONE]":
+                raw_json = line[6:]
+                try:
+                    event = json.loads(raw_json)
+                    delta = event.get("delta") if isinstance(event, dict) else None
+                    if (
+                        isinstance(delta, dict)
+                        and event.get("type") == "content_block_delta"
+                        and delta.get("type") == "text_delta"
+                        and isinstance(delta.get("text"), str)
+                    ):
+                        unmasked = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(
+                            delta["text"], pii_tokens
+                        )
+                        if unmasked != delta["text"]:
+                            event["delta"]["text"] = unmasked
+                            line = "data: " + json.dumps(event, ensure_ascii=False)
+                except (json.JSONDecodeError, KeyError, TypeError):
+                    pass
+            result_lines.append(line)
+
+        return "\n".join(result_lines).encode("utf-8")
+
     async def _stream_pii_unmasking(
         self,
         response: Any,
@@ -1237,13 +1270,19 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         from litellm.main import stream_chunk_builder
         from litellm.types.utils import ModelResponse
 
+        metadata = (request_data.get("metadata") or {}) if request_data else {}
+        pii_tokens: Dict[str, str] = metadata.get("pii_tokens", {})
+
         remaining_chunks: List[ModelResponseStream] = []
         try:
             async for chunk in response:
                 if isinstance(chunk, ModelResponseStream):
                     remaining_chunks.append(chunk)
                 elif isinstance(chunk, bytes):
-                    yield chunk  # type: ignore[misc]
+                    if pii_tokens:
+                        yield self._unmask_sse_bytes_chunk(chunk, pii_tokens)  # type: ignore[misc]
+                    else:
+                        yield chunk  # type: ignore[misc]
                     continue
 
             if not remaining_chunks:

litellm/proxy/hooks/batch_rate_limiter.py

@@ -531,11 +531,17 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             # Check if this is a managed file (base64 encoded unified file ID)
             from litellm.proxy.openai_files_endpoints.common_utils import (
                 _is_base64_encoded_unified_file_id,
+                get_models_from_unified_file_id,
             )
 
             # Managed files require bypassing the HTTP endpoint (which runs access-check hooks)
             # and calling the managed files hook directly with the user's credentials.
             is_managed_file = _is_base64_encoded_unified_file_id(file_id)
+            target_model_names = (
+                get_models_from_unified_file_id(is_managed_file)
+                if is_managed_file
+                else []
+            )
             if is_managed_file and user_api_key_dict is not None:
                 file_content = await self._fetch_managed_file_content(
                     file_id=file_id,
@@ -573,6 +579,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 await self._enforce_batch_file_model_access(
                     user_api_key_dict=user_api_key_dict,
                     file_content_as_dict=file_content_as_dict,
+                    target_model_names=target_model_names or None,
                 )
 
             input_file_usage = _get_batch_job_input_file_usage(
@@ -608,9 +615,13 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         self,
         user_api_key_dict: UserAPIKeyAuth,
         file_content_as_dict: List[dict],
+        target_model_names: Optional[List[str]] = None,
     ) -> None:
-        """Reject the batch if the caller is not authorized for every
-        ``body.model`` named inside the JSONL.
+        """Reject the batch if the caller is not authorized for the upload target.
+
+        For managed files, ``target_model_names`` (from the unified file id) is
+        the proxy alias the file was uploaded for and is used directly for auth.
+        For legacy/non-managed files, falls back to ``body.model`` values in the JSONL.
 
         Reuses standard auth helpers so the same model access rules the proxy
         enforces on `/chat/completions` apply here.
@@ -627,9 +638,12 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         from litellm.proxy.proxy_server import proxy_logging_obj
         from litellm.proxy.proxy_server import user_api_key_cache
 
-        models = _get_models_from_batch_input_file_content(file_content_as_dict)
-        if not models:
-            return
+        if target_model_names:
+            models = target_model_names
+        else:
+            models = _get_models_from_batch_input_file_content(file_content_as_dict)
+            if not models:
+                return
 
         team_object = None
         if (
@@ -660,12 +674,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
 
         llm_model_list = llm_router.model_list if llm_router is not None else None
         for model in models:
-            # body.model may be the provider id after replace_model_in_jsonl; map to proxy model_name for auth.
             model_to_check = model
-            if llm_router is not None:
-                proxy_model_name = llm_router.resolve_model_name_from_model_id(model)
-                if proxy_model_name is not None:
-                    model_to_check = proxy_model_name
             try:
                 if team_object is not None:
                     try:

litellm/proxy/hooks/litellm_skills/main.py

@@ -19,7 +19,7 @@ Usage:
     response = await litellm.acompletion(
         model="gpt-4o-mini",
         messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
-        container={"skills": [{"skill_id": "litellm:skill_abc123"}]},
+        container={"skills": [{"skill_id": "litellm_skill_abc123"}]},
     )
     # Response includes file_ids for generated files
 """
@@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional, Union
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.llms.litellm_proxy.skills.constants import LITELLM_SKILL_ID_PREFIX
 from litellm.llms.litellm_proxy.skills.prompt_injection import (
     SkillPromptInjectionHandler,
 )
@@ -43,7 +44,7 @@ class SkillsInjectionHook(CustomLogger):
     Pre/Post-call hook that processes skills from container.skills parameter.
 
     Pre-call (async_pre_call_hook):
-    - Skills with 'litellm:' prefix are fetched from LiteLLM DB
+    - Skills with 'litellm_skill_' prefix are fetched from LiteLLM DB
     - For Anthropic models: native skills pass through, LiteLLM skills converted to tools
     - For non-Anthropic models: LiteLLM skills are converted to tools + execute_code tool
 
@@ -78,7 +79,7 @@ class SkillsInjectionHook(CustomLogger):
         Process skills from container.skills before the LLM call.
 
         1. Check if container.skills exists in request
-        2. Separate skills by prefix (litellm: vs native)
+        2. Separate skills by prefix (litellm_skill_ vs native)
         3. Fetch LiteLLM skills from database
         4. For Anthropic: keep native skills in container
         5. For non-Anthropic: convert LiteLLM skills to tools, inject content, add execute_code
@@ -108,7 +109,7 @@ class SkillsInjectionHook(CustomLogger):
                 continue
 
             skill_id = skill.get("skill_id", "")
-            if skill_id.startswith("litellm_"):
+            if skill_id.startswith(LITELLM_SKILL_ID_PREFIX):
                 # Fetch from LiteLLM DB
                 db_skill = await self._fetch_skill_from_db(
                     skill_id,
@@ -287,7 +288,7 @@ class SkillsInjectionHook(CustomLogger):
         Fetch a skill from the LiteLLM database.
 
         Args:
-            skill_id: The skill ID (without 'litellm:' prefix)
+            skill_id: The skill ID (including the 'litellm_skill_' prefix)
 
         Returns:
             LiteLLM_SkillsTable or None if not found
@@ -382,10 +383,10 @@ class SkillsInjectionHook(CustomLogger):
         has_executable_tool = False
         for tc in tool_calls:
             tool_name = tc.get("name", "")
-            # Execute if it's litellm_code_execution OR a skill tool (skill_xxx)
+            # Execute if it's litellm_code_execution OR a skill tool (litellm_skill_xxx)
             if (
                 tool_name == LiteLLMInternalTools.CODE_EXECUTION.value
-                or tool_name.startswith("skill_")
+                or tool_name.startswith(LITELLM_SKILL_ID_PREFIX)
             ):
                 has_executable_tool = True
                 break
@@ -543,7 +544,7 @@ class SkillsInjectionHook(CustomLogger):
                     result = await self._execute_code(
                         code, skill_files, executor, generated_files
                     )
-                elif tool_name.startswith("skill_"):
+                elif tool_name.startswith(LITELLM_SKILL_ID_PREFIX):
                     # Skill tool - execute the skill's code
                     result = await self._execute_skill_tool(
                         tool_name, tool_input, skill_files, executor, generated_files

litellm/proxy/hooks/parallel_request_limiter.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel
 from typing_extensions import TypedDict
 
 import litellm
-from litellm import DualCache, ModelResponse
+from litellm import DualCache, EmbeddingResponse, ModelResponse, TextCompletionResponse
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
@@ -570,7 +570,9 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
 
             total_tokens = 0
 
-            if isinstance(response_obj, ModelResponse):
+            if isinstance(
+                response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
+            ):
                 total_tokens = response_obj.usage.total_tokens  # type: ignore
 
             # ------------
@@ -659,7 +661,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_user_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (
@@ -692,7 +697,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_team_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (
@@ -725,7 +733,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_end_user_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -39,7 +39,13 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import (
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
 from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
-from litellm.types.utils import CallTypes, ModelResponse, Usage
+from litellm.types.utils import (
+    CallTypes,
+    EmbeddingResponse,
+    ModelResponse,
+    TextCompletionResponse,
+    Usage,
+)
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -2736,9 +2742,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
         # Get total tokens from response
         total_tokens = 0
-        # spot fix for /responses api
-        if isinstance(response_obj, ModelResponse) or isinstance(
-            response_obj, BaseLiteLLMOpenAIResponseObject
+        if isinstance(
+            response_obj,
+            (
+                ModelResponse,
+                EmbeddingResponse,
+                TextCompletionResponse,
+                BaseLiteLLMOpenAIResponseObject,
+            ),
         ):
             _usage = getattr(response_obj, "usage", None)
             total_tokens = self._get_total_tokens_from_usage(

litellm/proxy/image_endpoints/endpoints.py

@@ -173,6 +173,16 @@ async def image_generation(
             )
         )
 
+        # Call response headers hook (matches base_process_llm_request behavior)
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            fastapi_response.headers.update(callback_headers)
+
         return response
     except Exception as e:
         await proxy_logging_obj.post_call_failure_hook(

litellm/proxy/management_endpoints/cache_settings_endpoints.py

@@ -27,6 +27,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.repositories.table_repositories import CacheConfigRepository
 from litellm.types.management_endpoints import (
     CACHE_SETTINGS_FIELDS,
@@ -160,8 +161,12 @@ class CacheSettingsManager:
         import json
 
         try:
-            cache_config = await CacheConfigRepository(prisma_client).table.find_unique(
-                where={"id": "cache_config"}
+            cache_config = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: CacheConfigRepository(prisma_client).table.find_unique(
+                    where={"id": "cache_config"}
+                ),
+                reason="init_cache_settings_in_db_lookup_failure",
             )
             if cache_config is not None and cache_config.cache_settings:
                 # Parse cache settings JSON

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -1850,11 +1850,10 @@ async def prepare_key_update_data(
 
     if "budget_duration" in non_default_values:
         budget_duration = non_default_values.pop("budget_duration")
-        if (
-            budget_duration
-            and (isinstance(budget_duration, str))
-            and len(budget_duration) > 0
-        ):
+        if budget_duration is None:
+            non_default_values["budget_duration"] = None
+            non_default_values["budget_reset_at"] = None
+        elif isinstance(budget_duration, str) and len(budget_duration) > 0:
             from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 
             key_reset_at = get_budget_reset_time(budget_duration=budget_duration)
@@ -2518,7 +2517,7 @@ async def update_key_fn(  # noqa: PLR0915
                 },
             )
 
-        data_json: dict = data.model_dump(exclude_unset=True, exclude_none=True)
+        data_json: dict = data.model_dump(exclude_unset=True)
         key = data_json.pop("key")
 
         # get the row from db
@@ -2588,6 +2587,17 @@ async def update_key_fn(  # noqa: PLR0915
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        if data.spend is not None:
+            try:
+                from litellm.proxy.proxy_server import _invalidate_spend_counter
+
+                token_to_invalidate = _hash_token_if_needed(key)
+                await _invalidate_spend_counter(
+                    counter_key=f"spend:key:{token_to_invalidate}"
+                )
+            except Exception:
+                pass
+
         asyncio.create_task(
             KeyManagementEventHooks.async_key_updated_hook(
                 data=data,
@@ -4775,6 +4785,13 @@ async def reset_key_spend_fn(
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        try:
+            from litellm.proxy.proxy_server import _invalidate_spend_counter
+
+            await _invalidate_spend_counter(counter_key=f"spend:key:{hashed_api_key}")
+        except Exception:
+            pass
+
         max_budget = updated_key.max_budget
         budget_reset_at = updated_key.budget_reset_at
 

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -21,7 +21,7 @@ import json
 import os
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, Iterable, List, Literal, Optional
+from typing import Any, Dict, Iterable, List, Literal, Optional, Set
 
 from fastapi import (
     APIRouter,
@@ -1722,11 +1722,13 @@ if MCP_AVAILABLE:
                     status_code=status.HTTP_403_FORBIDDEN,
                     detail={"error": f"Access denied to MCP server {server_id}"},
                 )
-            allowed_server_ids = (
-                await global_mcp_server_manager.get_allowed_mcp_servers(
-                    user_api_key_dict
+            allowed_server_ids: Set[str] = set()
+            for auth_context in await build_effective_auth_contexts(user_api_key_dict):
+                allowed_server_ids.update(
+                    await global_mcp_server_manager.get_allowed_mcp_servers(
+                        auth_context
+                    )
                 )
-            )
             if server.server_id not in allowed_server_ids:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -357,6 +357,42 @@ class TeamMemberBudgetHandler:
         data_dict.pop("team_member_rpm_limit", None)
         data_dict.pop("team_member_tpm_limit", None)
 
+    @staticmethod
+    async def clear_team_member_budget_fields(
+        team_table: LiteLLM_TeamTable,
+        user_api_key_dict: "UserAPIKeyAuth",
+        updated_kv: dict,
+        explicitly_set_fields: set,
+    ) -> dict:
+        """Clear explicitly-nulled fields on the team member budget row."""
+        from litellm.proxy._types import BudgetNewRequest
+        from litellm.proxy.management_endpoints.budget_management_endpoints import (
+            update_budget,
+        )
+
+        if team_table.metadata is None:
+            team_table.metadata = {}
+
+        team_member_budget_id = team_table.metadata.get("team_member_budget_id")
+        if team_member_budget_id is not None and isinstance(team_member_budget_id, str):
+            budget_request = BudgetNewRequest(budget_id=team_member_budget_id)
+            if "team_member_budget" in explicitly_set_fields:
+                budget_request.max_budget = None
+            if "team_member_budget_duration" in explicitly_set_fields:
+                budget_request.budget_duration = None
+                budget_request.budget_reset_at = None
+            if "team_member_rpm_limit" in explicitly_set_fields:
+                budget_request.rpm_limit = None
+            if "team_member_tpm_limit" in explicitly_set_fields:
+                budget_request.tpm_limit = None
+            await update_budget(
+                budget_obj=budget_request,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
+        return updated_kv
+
     @staticmethod
     async def backfill_team_member_budget_entries(
         team_id: str,
@@ -779,55 +815,40 @@ async def _check_user_team_limits(
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
     user_api_key_cache: Any,
-    existing_team_max_budget: Optional[float] = None,
 ) -> None:
     """
-    Check user team limits for standalone teams (not org-scoped).
+    Enforce the caller's personal limits when CREATING a standalone team.
 
-    This validates:
-    - Team budget vs user's max_budget
-    - Team models vs user's allowed models
+    This validates the requested team budget / models / tpm / rpm against the
+    caller's own limits, so a non-admin user cannot mint a brand-new team that
+    is richer than themselves.
 
-    Should only be called for standalone teams (when organization_id is None).
-    For org-scoped teams, use _check_org_team_limits() instead.
-
-    `existing_team_max_budget` is the team's current `max_budget` on the
-    /team/update path. When the incoming `max_budget` is unchanged or lower
-    than the team's current budget, the personal-budget comparison is skipped
-    so a team admin can edit other fields (e.g. tpm_limit, team name) without
-    being blocked by a budget the team already has. The UI sends the full team
-    object on every update, so the unchanged `max_budget` would otherwise fail.
+    Only used by /team/new for standalone teams (organization_id is None).
+    /team/update does NOT call this — an existing team's admin is already
+    authorized via _verify_team_access() and is not gated by their personal
+    wallet. Org-scoped teams use _check_org_team_limits() instead.
     """
     # Validate team budget against user's max_budget
     if data.max_budget is not None and user_api_key_dict.user_id is not None:
-        # On /team/update, allow unchanged or lower budgets without checking
-        # the caller's personal max_budget. Only increases above the team's
-        # current budget are validated against the user's personal limit.
-        budget_unchanged_or_lower = (
-            existing_team_max_budget is not None
-            and data.max_budget <= existing_team_max_budget
+        user_obj = await get_user_object(
+            user_id=user_api_key_dict.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
         )
 
-        if not budget_unchanged_or_lower:
-            user_obj = await get_user_object(
-                user_id=user_api_key_dict.user_id,
-                prisma_client=prisma_client,
-                user_api_key_cache=user_api_key_cache,
-                user_id_upsert=False,
+        if (
+            user_obj is not None
+            and user_obj.max_budget is not None
+            and data.max_budget > user_obj.max_budget
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": f"max budget higher than user max. User max budget={user_obj.max_budget}. User role={user_api_key_dict.user_role}"
+                },
             )
 
-            if (
-                user_obj is not None
-                and user_obj.max_budget is not None
-                and data.max_budget > user_obj.max_budget
-            ):
-                raise HTTPException(
-                    status_code=400,
-                    detail={
-                        "error": f"max budget higher than user max. User max budget={user_obj.max_budget}. User role={user_api_key_dict.user_role}"
-                    },
-                )
-
     # Validate team models against user's allowed models
     if data.models is not None and len(user_api_key_dict.models) > 0:
         for m in data.models:
@@ -865,6 +886,45 @@ async def _check_user_team_limits(
         )
 
 
+def _check_team_budget_update_authority(
+    data: UpdateTeamRequest,
+    user_api_key_dict: UserAPIKeyAuth,
+    existing_team_max_budget: Optional[float],
+) -> None:
+    """
+    Restrict who can grow a standalone team's spend ceiling on /team/update.
+
+    A team admin (already authorized via _verify_team_access) may keep or lower
+    the team budget, but only a proxy admin may grow it - by raising max_budget
+    above the team's current value or by removing the cap (setting it to None).
+    Setting a finite budget on a team that has no cap is a restriction and is
+    allowed. Org-scoped teams are governed by _check_org_team_limits().
+    """
+    if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return
+    if existing_team_max_budget is None:
+        return
+
+    budget_explicitly_set = "max_budget" in (
+        getattr(data, "model_fields_set", None) or set()
+    )
+    if budget_explicitly_set and data.max_budget is None:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": f"Only a proxy admin can remove a team's max_budget. Team's current max_budget={existing_team_max_budget}."
+            },
+        )
+
+    if data.max_budget is not None and data.max_budget > existing_team_max_budget:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": f"Only a proxy admin can raise a team's max_budget. Team's current max_budget={existing_team_max_budget}, requested={data.max_budget}."
+            },
+        )
+
+
 #### TEAM MANAGEMENT ####
 @router.post(
     "/team/new",
@@ -1827,22 +1887,14 @@ async def update_team(  # noqa: PLR0915
                     prisma_client=prisma_client,
                 )
 
-        # Check user limits for standalone teams (not org-scoped)
-        # Skip for PROXY_ADMIN users
-        if (
-            user_api_key_dict.user_role is None
-            or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
-        ):
-            # Only validate user budget/models for standalone teams
-            # For org-scoped teams, validation is done by _check_org_team_limits() above
-            if org_id_to_check is None:
-                await _check_user_team_limits(
-                    data=data,
-                    user_api_key_dict=user_api_key_dict,
-                    prisma_client=prisma_client,
-                    user_api_key_cache=user_api_key_cache,
-                    existing_team_max_budget=existing_team_row.max_budget,
-                )
+        # Only a proxy admin may grow a standalone team's spend ceiling.
+        # Org-scoped teams are validated by _check_org_team_limits() above.
+        if org_id_to_check is None:
+            _check_team_budget_update_authority(
+                data=data,
+                user_api_key_dict=user_api_key_dict,
+                existing_team_max_budget=existing_team_row.max_budget,
+            )
 
         updated_kv = data.json(exclude_unset=True)
 
@@ -1856,11 +1908,25 @@ async def update_team(  # noqa: PLR0915
         # Check budget_duration and budget_reset_at
         _set_budget_reset_at(data, updated_kv)
 
-        if TeamMemberBudgetHandler.should_create_budget(
-            team_member_budget=data.team_member_budget,
-            team_member_rpm_limit=data.team_member_rpm_limit,
-            team_member_tpm_limit=data.team_member_tpm_limit,
-            team_member_budget_duration=data.team_member_budget_duration,
+        _team_member_fields_in_request = {
+            field
+            for field in [
+                "team_member_budget",
+                "team_member_rpm_limit",
+                "team_member_tpm_limit",
+                "team_member_budget_duration",
+            ]
+            if field in updated_kv
+        }
+
+        if (
+            _team_member_fields_in_request
+            and TeamMemberBudgetHandler.should_create_budget(
+                team_member_budget=data.team_member_budget,
+                team_member_rpm_limit=data.team_member_rpm_limit,
+                team_member_tpm_limit=data.team_member_tpm_limit,
+                team_member_budget_duration=data.team_member_budget_duration,
+            )
         ):
             updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table(
                 team_table=existing_team_row,
@@ -1883,6 +1949,13 @@ async def update_team(  # noqa: PLR0915
                     team_member_budget_id=_backfill_budget_id,
                     prisma_client=prisma_client,
                 )
+        elif _team_member_fields_in_request:
+            updated_kv = await TeamMemberBudgetHandler.clear_team_member_budget_fields(
+                team_table=existing_team_row,
+                user_api_key_dict=user_api_key_dict,
+                updated_kv=updated_kv,
+                explicitly_set_fields=_team_member_fields_in_request,
+            )
         else:
             TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
 
@@ -1971,6 +2044,8 @@ def _set_budget_reset_at(data: UpdateTeamRequest, updated_kv: dict) -> None:
 
         reset_at = get_budget_reset_time(budget_duration=data.budget_duration)
         updated_kv["budget_reset_at"] = reset_at
+    elif "budget_duration" in updated_kv and updated_kv["budget_duration"] is None:
+        updated_kv["budget_reset_at"] = None
 
     if data.budget_limits is not None and len(data.budget_limits) > 0:
         from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -426,12 +426,7 @@ async def mistral_proxy_route(
     )
 
     ## check for streaming
-    is_streaming_request = False
-    # anthropic is streaming when 'stream' = True is in the body
-    if request.method == "POST":
-        _request_body = await request.json()
-        if _request_body.get("stream"):
-            is_streaming_request = True
+    is_streaming_request = await is_streaming_request_fn(request)
 
     ## CREATE PASS-THROUGH
     endpoint_func = create_pass_through_route(

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -5,7 +5,7 @@ Handles cost tracking and logging for OpenAI passthrough endpoints, specifically
 """
 
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple, Union
 from urllib.parse import urlparse
 
 import httpx
@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import (
 )
 from litellm.llms.openai.openai import OpenAIConfig
 from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import (
     BasePassthroughLoggingHandler,
@@ -29,6 +30,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     EndpointType,
     PassthroughStandardLoggingPayload,
 )
+from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.utils import ImageResponse, LlmProviders, PassthroughCallTypes
 from litellm.utils import ModelResponse, TextCompletionResponse
 
@@ -236,6 +238,42 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
             )
             return 0.0
 
+    @staticmethod
+    def _build_responses_api_response_and_cost(
+        model: str,
+        httpx_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        custom_llm_provider: str,
+    ) -> Tuple[ResponsesAPIResponse, float]:
+        """Transform a Responses API raw response into a ResponsesAPIResponse
+        and compute its cost.
+
+        The Responses API has a different on-the-wire shape from chat
+        completions (`output: [...]` instead of `choices: [...]`), so the
+        chat-completions `transform_response` raises KeyError 'choices' on
+        a Responses payload. Use the dedicated Responses-API transformer
+        (`OpenAIResponsesAPIConfig.transform_response_api_response`) here.
+
+        Returns (litellm_model_response, response_cost) — symmetric with the
+        chat-completions branch which produces the same two values inline,
+        and analogous to the image branches' `_calculate_image_*_cost` helpers
+        (which return cost only because the image-response object is trivial
+        to build inline; the Responses payload needs a real transformer).
+        """
+        responses_config = OpenAIResponsesAPIConfig()
+        litellm_model_response = responses_config.transform_response_api_response(
+            model=model,
+            raw_response=httpx_response,
+            logging_obj=logging_obj,
+        )
+        response_cost = litellm.completion_cost(
+            completion_response=litellm_model_response,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            call_type="responses",
+        )
+        return litellm_model_response, response_cost
+
     @staticmethod
     def openai_passthrough_handler(  # noqa: PLR0915
         httpx_response: httpx.Response,
@@ -301,7 +339,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
         try:
             response_cost = 0.0
             litellm_model_response: Optional[
-                Union[ModelResponse, TextCompletionResponse, ImageResponse]
+                Union[
+                    ModelResponse,
+                    TextCompletionResponse,
+                    ImageResponse,
+                    ResponsesAPIResponse,
+                ]
             ] = None
             handler_instance = OpenAIPassthroughLoggingHandler()
 
@@ -384,29 +427,18 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     litellm_model_response._hidden_params = {}
                 litellm_model_response._hidden_params["response_cost"] = response_cost
             elif is_responses:
-                # Handle responses API cost calculation
-                provider_config = handler_instance.get_provider_config(model=model)
-                existing_litellm_params = kwargs.get("litellm_params", {}) or {}
-                litellm_model_response = provider_config.transform_response(
-                    raw_response=httpx_response,
-                    model_response=litellm.ModelResponse(),
+                # Responses-API cost tracking — see
+                # `_build_responses_api_response_and_cost` for why this needs
+                # a dedicated transformer (the chat-completions transform
+                # crashes on the Responses payload shape).
+                (
+                    litellm_model_response,
+                    response_cost,
+                ) = OpenAIPassthroughLoggingHandler._build_responses_api_response_and_cost(
                     model=model,
-                    messages=request_body.get("messages", []),
+                    httpx_response=httpx_response,
                     logging_obj=logging_obj,
-                    optional_params=request_body.get("optional_params", {}),
-                    api_key="",
-                    request_data=request_body,
-                    encoding=litellm.encoding,
-                    json_mode=False,
-                    litellm_params=existing_litellm_params,
-                )
-
-                # Calculate cost using LiteLLM's cost calculator with responses call type
-                response_cost = litellm.completion_cost(
-                    completion_response=litellm_model_response,
-                    model=model,
                     custom_llm_provider=custom_llm_provider,
-                    call_type="responses",
                 )
 
             # Update kwargs with cost information

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -1099,6 +1099,20 @@ async def pass_through_request(  # noqa: PLR0915
                     status_code=e.response.status_code, detail=await e.response.aread()
                 )
 
+            # Call response headers hook for streaming pass-through
+            _response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
+                headers=response.headers,
+                litellm_call_id=litellm_call_id,
+            )
+            callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+                data=_parsed_body or {},
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_headers=dict(request.headers),
+            )
+            if callback_headers:
+                _response_headers.update(callback_headers)
+
             return StreamingResponse(
                 PassThroughStreamingHandler.chunk_processor(
                     response=response,
@@ -1109,10 +1123,7 @@ async def pass_through_request(  # noqa: PLR0915
                     passthrough_success_handler_obj=pass_through_endpoint_logging,
                     url_route=str(url),
                 ),
-                headers=HttpPassThroughEndpointHelpers.get_response_headers(
-                    headers=response.headers,
-                    litellm_call_id=litellm_call_id,
-                ),
+                headers=_response_headers,
                 status_code=response.status_code,
             )
 
@@ -1151,6 +1162,20 @@ async def pass_through_request(  # noqa: PLR0915
                     status_code=e.response.status_code, detail=await e.response.aread()
                 )
 
+            # Call response headers hook for detected streaming pass-through
+            _response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
+                headers=response.headers,
+                litellm_call_id=litellm_call_id,
+            )
+            callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+                data=_parsed_body or {},
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_headers=dict(request.headers),
+            )
+            if callback_headers:
+                _response_headers.update(callback_headers)
+
             return StreamingResponse(
                 PassThroughStreamingHandler.chunk_processor(
                     response=response,
@@ -1161,10 +1186,7 @@ async def pass_through_request(  # noqa: PLR0915
                     passthrough_success_handler_obj=pass_through_endpoint_logging,
                     url_route=str(url),
                 ),
-                headers=HttpPassThroughEndpointHelpers.get_response_headers(
-                    headers=response.headers,
-                    litellm_call_id=litellm_call_id,
-                ),
+                headers=_response_headers,
                 status_code=response.status_code,
             )
 
@@ -1303,6 +1325,16 @@ async def pass_through_request(  # noqa: PLR0915
             api_base=str(url._uri_reference),
         )
 
+        # Call response headers hook
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=_parsed_body or {},
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            custom_headers.update(callback_headers)
+
         response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
             headers=response.headers,
             custom_headers=custom_headers,

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -458,7 +458,16 @@ class PassThroughEndpointLogging:
         return False
 
     def _is_supported_openai_endpoint(self, url_route: str) -> bool:
-        """Check if the OpenAI endpoint is supported by the passthrough logging handler."""
+        """Check if the OpenAI endpoint is supported by the passthrough logging handler.
+
+        The Responses API route is included because
+        `openai_passthrough_handler` has a dedicated `elif is_responses:`
+        branch that knows how to extract usage + cost from the
+        Responses-API on-the-wire shape. Without including it here, the
+        outer dispatch filters Responses calls out before reaching the
+        handler — the inner branch is then unreachable and Responses
+        calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
+        """
         from .llm_provider_handlers.openai_passthrough_logging_handler import (
             OpenAIPassthroughLoggingHandler,
         )
@@ -469,6 +478,7 @@ class PassThroughEndpointLogging:
                 url_route
             )
             or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
         )
 
     def _set_cost_per_request(

litellm/proxy/proxy_cli.py

@@ -555,6 +555,7 @@ class ProxyInitializationHelpers:
 
 
 @click.command()
+@click.argument("cli_args", nargs=-1)
 @click.option(
     "--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST"
 )
@@ -808,6 +809,7 @@ class ProxyInitializationHelpers:
     help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.",
 )
 def run_server(  # noqa: PLR0915
+    cli_args,
     host,
     port,
     api_base,
@@ -854,6 +856,20 @@ def run_server(  # noqa: PLR0915
     use_v2_migration_resolver: bool,
     reload: bool,
 ):
+    if cli_args:
+        if cli_args == ("xai-oauth", "login"):
+            from litellm.llms.xai.oauth import XAIOAuthAuthenticator
+
+            authenticator = XAIOAuthAuthenticator()
+            auth_data = authenticator.login()
+            click.echo(
+                f"xAI OAuth login successful. Credentials saved to {authenticator.auth_file}."
+            )
+            if auth_data.get("expires_at"):
+                click.echo(f"Access token expires at {auth_data['expires_at']}.")
+            return
+        raise click.UsageError(f"Unknown command: {' '.join(cli_args)}")
+
     if setup:
         from litellm.setup_wizard import run_setup_wizard
 

litellm/proxy/proxy_server.py

@@ -311,7 +311,10 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.container_endpoints.endpoints import router as container_router
 from litellm.proxy.credential_endpoints.endpoints import router as credential_router
 from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
-from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
+from litellm.proxy.db.exception_handler import (
+    PrismaDBExceptionHandler,
+    call_with_db_reconnect_retry,
+)
 from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
 from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
@@ -2263,7 +2266,7 @@ async def _reconcile_budget_reservation_for_counter_update(
         )
     except Exception:
         verbose_proxy_logger.warning(
-            "Failed to reconcile budget reservation after persisted spend; invalidating reserved counters and continuing",
+            "Failed to reconcile budget reservation after persisted spend; invalidating reserved counters and falling back to direct increment",
             exc_info=True,
         )
         try:
@@ -2274,6 +2277,7 @@ async def _reconcile_budget_reservation_for_counter_update(
             verbose_proxy_logger.exception(
                 "Failed to invalidate reserved counters after reservation reconciliation failed"
             )
+        return set()
     return reserved_counter_keys
 
 
@@ -4089,6 +4093,8 @@ class ProxyConfig:
                     verbose_proxy_logger.debug(
                         f"litellm.post_call_rules: {litellm.post_call_rules}"
                     )
+                elif key == "max_budget":
+                    litellm.max_budget = float(value)
                 elif key == "max_internal_user_budget":
                     litellm.max_internal_user_budget = float(value)  # type: ignore
                 elif key == "default_max_internal_user_budget":
@@ -5982,8 +5988,12 @@ class ProxyConfig:
         """
 
         try:
-            sso_settings = await SSOConfigRepository(prisma_client).table.find_unique(
-                where={"id": "sso_config"}
+            sso_settings = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: SSOConfigRepository(prisma_client).table.find_unique(
+                    where={"id": "sso_config"}
+                ),
+                reason="init_sso_settings_in_db_lookup_failure",
             )
             if sso_settings is not None:
                 sso_settings.sso_settings.pop("role_mappings", None)
@@ -6018,9 +6028,13 @@ class ProxyConfig:
         )
 
         try:
-            db_record = await ConfigOverridesRepository(
-                prisma_client
-            ).table.find_unique(where={"config_type": "hashicorp_vault"})
+            db_record = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
+                    where={"config_type": "hashicorp_vault"}
+                ),
+                reason="init_hashicorp_vault_config_override_lookup_failure",
+            )
 
             if db_record is None or db_record.config_value is None:
                 if self._last_hashicorp_vault_config is not None:
@@ -9191,6 +9205,16 @@ async def audio_speech(
             hidden_params=hidden_params,
         )
 
+        # Call response headers hook (matches audio_transcription behavior)
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            custom_headers.update(callback_headers)
+
         # Determine media type based on model type
         media_type = "audio/mpeg"  # Default for OpenAI TTS
         request_model = data.get("model", "")

litellm/proxy/search_endpoints/search_tool_registry.py

@@ -7,6 +7,7 @@ from typing import List, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.table_repositories import SearchToolsRepository
 from litellm.types.search import SearchTool
@@ -180,10 +181,12 @@ class SearchToolRegistry:
             List of search tool configurations
         """
         try:
-            search_tools_from_db = await SearchToolsRepository(
-                prisma_client
-            ).table.find_many(
-                order={"created_at": "desc"},
+            search_tools_from_db = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: SearchToolsRepository(prisma_client).table.find_many(
+                    order={"created_at": "desc"},
+                ),
+                reason="get_all_search_tools_from_db_lookup_failure",
             )
 
             search_tools: List[SearchTool] = []

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -3405,7 +3405,7 @@ async def ui_view_session_spend_logs(
                 session_id, status, mcp_namespaced_tool_name, agent_id
             FROM "LiteLLM_SpendLogs"
             WHERE session_id = $1
-            ORDER BY "startTime" ASC
+            ORDER BY "startTime" DESC
             LIMIT $2 OFFSET $3
         """
         result = await prisma_client.db.query_raw(

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -178,6 +178,11 @@ class UISettings(BaseModel):
         description="If true, org admins cannot generate API keys via /key/generate.",
     )
 
+    disable_ui_nudges: bool = Field(
+        default=False,
+        description="If true, suppresses in-product UI nudges (survey and Claude Code feedback popups) for all users.",
+    )
+
 
 class UISettingsResponse(SettingsResponse):
     """Response model for UI settings"""
@@ -201,6 +206,7 @@ ALLOWED_UI_SETTINGS_FIELDS = {
     "scope_user_search_to_org",
     "disable_custom_api_keys",
     "disable_key_generate_for_org_admin",
+    "disable_ui_nudges",
 }
 
 # Flags that must be synced from the persisted UISettings into

litellm/proxy/utils.py

@@ -2496,7 +2496,8 @@ class ProxyLogging:
         )
 
         return {
-            "custom_llm_provider": hidden_params.get("custom_llm_provider"),
+            "custom_llm_provider": hidden_params.get("custom_llm_provider")
+            or getattr(response, "custom_llm_provider", None),
             "model_info": model_info,
             "api_base": hidden_params.get("api_base"),
             "model_id": hidden_params.get("model_id"),

litellm/router.py

@@ -1658,6 +1658,67 @@ class Router:
                     f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
                 )
 
+    def _add_encrypted_content_affinity_check(
+        self, enable_global_affinity: bool
+    ) -> None:
+        from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
+            EncryptedContentAffinityCheck,
+        )
+
+        def _move_before_deployment_affinity(
+            callback_list: List[Any],
+            callback_to_move: EncryptedContentAffinityCheck,
+        ) -> None:
+            if callback_to_move not in callback_list:
+                return
+            callback_list.remove(callback_to_move)
+            insert_index = next(
+                (
+                    idx
+                    for idx, callback in enumerate(callback_list)
+                    if isinstance(callback, DeploymentAffinityCheck)
+                ),
+                len(callback_list),
+            )
+            callback_list.insert(insert_index, callback_to_move)
+
+        if (
+            enable_global_affinity
+            or EncryptedContentAffinityCheck.has_model_group_affinity_enabled(
+                self.model_group_affinity_config
+            )
+        ):
+            if self.optional_callbacks is None:
+                self.optional_callbacks = []
+
+            existing_ec_callback: Optional[EncryptedContentAffinityCheck] = None
+            for cb in self.optional_callbacks:
+                if isinstance(cb, EncryptedContentAffinityCheck):
+                    existing_ec_callback = cb
+                    break
+
+            if existing_ec_callback is not None:
+                existing_ec_callback.router = self
+                existing_ec_callback.enable_global_affinity = (
+                    existing_ec_callback.enable_global_affinity
+                    or enable_global_affinity
+                )
+                existing_ec_callback.model_group_affinity_config = (
+                    self.model_group_affinity_config or {}
+                )
+                ec_callback = existing_ec_callback
+            else:
+                ec_callback = EncryptedContentAffinityCheck(
+                    router=self,
+                    enable_global_affinity=enable_global_affinity,
+                    model_group_affinity_config=self.model_group_affinity_config,
+                )
+                self.optional_callbacks.append(ec_callback)
+                litellm.logging_callback_manager.add_litellm_callback(ec_callback)
+
+            _move_before_deployment_affinity(self.optional_callbacks, ec_callback)
+            _move_before_deployment_affinity(litellm.callbacks, ec_callback)
+
     def add_optional_pre_call_checks(
         self, optional_pre_call_checks: Optional[OptionalPreCallChecks]
     ):
@@ -1721,22 +1782,11 @@ class Router:
         # ---------------------------------------------------------------------
         # Encrypted content affinity
         # ---------------------------------------------------------------------
-        if "encrypted_content_affinity" in optional_pre_call_checks:
-            from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
-                EncryptedContentAffinityCheck,
+        self._add_encrypted_content_affinity_check(
+            enable_global_affinity=(
+                "encrypted_content_affinity" in optional_pre_call_checks
             )
-
-            if self.optional_callbacks is None:
-                self.optional_callbacks = []
-
-            already_registered = any(
-                isinstance(cb, EncryptedContentAffinityCheck)
-                for cb in self.optional_callbacks
-            )
-            if not already_registered:
-                ec_callback = EncryptedContentAffinityCheck(router=self)
-                self.optional_callbacks.append(ec_callback)
-                litellm.logging_callback_manager.add_litellm_callback(ec_callback)
+        )
 
         # ---------------------------------------------------------------------
         # Remaining optional pre-call checks
@@ -7738,6 +7788,39 @@ class Router:
 
         return hash_object.hexdigest()
 
+    @staticmethod
+    def _inherit_builtin_cache_pricing(
+        model_info: dict, backend_model: str, custom_llm_provider: Optional[str]
+    ) -> None:
+        """Fill missing cache pricing on a custom-priced deployment entry from
+        the backend model's built-in cost map entry, so a deployment that
+        only spells out ``input_cost_per_token``/``output_cost_per_token``
+        does not silently bill cache_read/cache_creation at 0.
+
+        User-specified cache fields always win; only ``None``/missing entries
+        are inherited. No-op when the backend model has no canonical entry.
+        """
+        cache_fields = (
+            "cache_creation_input_token_cost",
+            "cache_creation_input_token_cost_above_1hr",
+            "cache_creation_input_token_cost_above_200k_tokens",
+            "cache_read_input_token_cost",
+            "cache_read_input_token_cost_above_200k_tokens",
+        )
+        if all(model_info.get(f) is not None for f in cache_fields):
+            return
+        try:
+            backend_info = litellm.get_model_info(
+                model=backend_model, custom_llm_provider=custom_llm_provider
+            )
+        except Exception:
+            return
+        for field in cache_fields:
+            if model_info.get(field) is None:
+                backend_value = backend_info.get(field)
+                if backend_value is not None:
+                    model_info[field] = backend_value
+
     def _create_deployment(
         self,
         deployment_info: dict,
@@ -7766,6 +7849,13 @@ class Router:
                 if deployment.litellm_params.get(field) is not None:
                     _model_info[field] = deployment.litellm_params[field]
 
+            if _model_info.get("input_cost_per_token") is not None:
+                Router._inherit_builtin_cache_pricing(
+                    model_info=_model_info,
+                    backend_model=deployment.litellm_params.model,
+                    custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+                )
+
             ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP
             model_id = deployment.model_info.id
             if model_id is not None:
@@ -8471,6 +8561,13 @@ class Router:
                     credential_values.get("api_key")
                     or deployment.litellm_params.api_key
                 )
+                if api_key is None:
+                    verbose_router_logger.debug(
+                        "Skipping pass-through credential setup for deployment model=%s, custom_llm_provider=%s; no api_key set. Providers like bedrock resolve credentials at request time.",
+                        model,
+                        custom_llm_provider,
+                    )
+                    return
                 passthrough_endpoint_router.set_pass_through_credentials(
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
@@ -8505,6 +8602,13 @@ class Router:
             if field_value is not None:
                 _model_info_dict[field] = field_value
 
+        if _model_info_dict.get("input_cost_per_token") is not None:
+            Router._inherit_builtin_cache_pricing(
+                model_info=_model_info_dict,
+                backend_model=deployment.litellm_params.model,
+                custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+            )
+
         # Register custom pricing in litellm.model_cost.
         # Mirrors _create_deployment() logic to ensure dynamically-added deployments
         # (e.g., loaded from DB) also have their custom pricing registered.

litellm/router_utils/pre_call_checks/deployment_affinity_check.py

@@ -39,7 +39,12 @@ class DeploymentAffinityCheck(CustomLogger):
 
     CACHE_KEY_PREFIX = "deployment_affinity:v1"
     VALID_FLAGS = frozenset(
-        {"deployment_affinity", "responses_api_deployment_check", "session_affinity"}
+        {
+            "deployment_affinity",
+            "responses_api_deployment_check",
+            "session_affinity",
+            "encrypted_content_affinity",
+        }
     )
 
     def __init__(

litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py

@@ -37,7 +37,7 @@ Safe to enable globally:
 """
 
 import time
-from typing import TYPE_CHECKING, Any, List, Optional, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
 
 import httpx
 
@@ -64,17 +64,45 @@ class EncryptedContentAffinityCheck(CustomLogger):
     The ``model_id`` is decoded directly from the litellm-encoded item IDs –
     no caching or TTL management needed.
 
-    Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])``.
+    Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])`` or
+    per-model group ``model_group_affinity_config``.
     """
 
-    def __init__(self, router: Optional["Router"] = None) -> None:
+    def __init__(
+        self,
+        router: Optional["Router"] = None,
+        enable_global_affinity: bool = True,
+        model_group_affinity_config: Optional[Dict[str, List[str]]] = None,
+    ) -> None:
         super().__init__()
         self.router = router
+        self.enable_global_affinity = enable_global_affinity
+        self.model_group_affinity_config: Dict[str, List[str]] = (
+            model_group_affinity_config or {}
+        )
 
     # ------------------------------------------------------------------
     # Helpers
     # ------------------------------------------------------------------
 
+    @staticmethod
+    def has_model_group_affinity_enabled(
+        model_group_affinity_config: Optional[Dict[str, List[str]]],
+    ) -> bool:
+        if not model_group_affinity_config:
+            return False
+
+        return any(
+            "encrypted_content_affinity" in checks
+            for checks in model_group_affinity_config.values()
+        )
+
+    def _is_enabled_for_model_group(self, model_group: str) -> bool:
+        group_checks = self.model_group_affinity_config.get(model_group)
+        return self.enable_global_affinity or (
+            group_checks is not None and "encrypted_content_affinity" in group_checks
+        )
+
     @staticmethod
     def _extract_model_id_from_input(request_input: Any) -> Optional[str]:
         """
@@ -213,6 +241,8 @@ class EncryptedContentAffinityCheck(CustomLogger):
         """
         request_kwargs = request_kwargs or {}
         typed_healthy_deployments = cast(List[dict], healthy_deployments)
+        if not self._is_enabled_for_model_group(model):
+            return typed_healthy_deployments
 
         # Signal to the response post-processor that encrypted item IDs should be
         # encoded in the output of this request.  Only set the flag when

litellm/setup_wizard.py

@@ -52,11 +52,12 @@ PROVIDERS: List[Dict] = [
     {
         "id": "anthropic",
         "name": "Anthropic",
-        "description": "Claude Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5",
+        "description": "Claude Fable 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5",
         "env_key": "ANTHROPIC_API_KEY",
         "key_hint": "sk-ant-...",
         "test_model": "claude-haiku-4-5-20251001",
         "models": [
+            "claude-fable-5",
             "claude-opus-4-8",
             "claude-opus-4-7",
             "claude-opus-4-6",

litellm/types/caching.py

@@ -118,4 +118,5 @@ class CachedEmbedding(TypedDict):
     index: Optional[int]
     object: Optional[str]
     model: Optional[str]
+    prompt_tokens: Optional[int]
     prompt_tokens_details: Optional[dict]

litellm/types/router.py

@@ -220,6 +220,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     use_in_pass_through: Optional[bool] = False
     use_litellm_proxy: Optional[bool] = False
     use_chat_completions_api: Optional[bool] = None
+    use_xai_oauth: Optional[bool] = Field(
+        default=False,
+        description="Use stored xAI OAuth credentials when no xAI API key is configured.",
+    )
     model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
     merge_reasoning_content_in_choices: Optional[bool] = False
     model_info: Optional[Dict] = None

litellm/types/utils.py

@@ -197,6 +197,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     ]  # OpenAI priority service tier pricing
     cache_read_input_token_cost_above_200k_tokens: Optional[float]
     cache_read_input_token_cost_above_272k_tokens: Optional[float]
+    cache_read_input_token_cost_above_512k_tokens: Optional[float]
     input_cost_per_character: Optional[float]  # only for vertex ai models
     input_cost_per_audio_token: Optional[float]
     input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models
@@ -206,6 +207,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     input_cost_per_token_above_272k_tokens: Optional[
         float
     ]  # GPT-5.4/5.4-pro: prompts >272K priced at 2x input
+    input_cost_per_token_above_512k_tokens: Optional[
+        float
+    ]  # MiniMax-M3: prompts >512K priced at 2x input
     input_cost_per_character_above_128k_tokens: Optional[
         float
     ]  # only for vertex ai models
@@ -239,6 +243,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     output_cost_per_token_above_272k_tokens: Optional[
         float
     ]  # GPT-5.4/5.4-pro: prompts >272K priced at 1.5x output
+    output_cost_per_token_above_512k_tokens: Optional[
+        float
+    ]  # MiniMax-M3: prompts >512K priced at 2x output
     output_cost_per_character_above_128k_tokens: Optional[
         float
     ]  # only for vertex ai models
@@ -3217,6 +3224,7 @@ all_litellm_params = (
         "search_tool_name",
         "order",
         "enable_json_schema_validation",
+        "use_xai_oauth",
     ]
     + list(StandardCallbackDynamicParams.__annotations__.keys())
     + list(CustomPricingLiteLLMParams.model_fields.keys())

litellm/utils.py

@@ -2887,6 +2887,61 @@ def _convert_stringified_numbers(value):
     return value
 
 
+_BEDROCK_REGION_PREFIXES = (
+    "us.",
+    "eu.",
+    "apac.",
+    "jp.",
+    "au.",
+    "us-gov.",
+    "global.",
+    "ap-northeast-1.",
+)
+
+_CACHE_PRICING_FIELDS = (
+    "cache_creation_input_token_cost",
+    "cache_creation_input_token_cost_above_1hr",
+    "cache_creation_input_token_cost_above_200k_tokens",
+    "cache_read_input_token_cost",
+    "cache_read_input_token_cost_above_200k_tokens",
+)
+
+
+def _resolve_builtin_model_cost_entry(
+    key: str, provider: str
+) -> Optional[Dict[str, Any]]:
+    """Best-effort lookup of a built-in ``model_cost`` entry for a custom key
+    whose shape ``get_model_info`` cannot resolve (double provider prefixes
+    like ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases).
+
+    Returns a copy of the matching entry so the caller can inherit its defaults
+    (most importantly cache pricing) without mutating the shared built-in.
+    Returns ``None`` when no safe match exists.
+    """
+    candidates: List[str] = []
+    segments = key.split("/")
+    idx = 0
+    while idx < len(segments) - 1 and segments[idx] in LlmProvidersSet:
+        idx += 1
+        candidates.append("/".join(segments[idx:]))
+
+    base = candidates[-1] if candidates else key
+    for region_prefix in _BEDROCK_REGION_PREFIXES:
+        if base.startswith(region_prefix):
+            candidates.append(base[len(region_prefix) :])
+
+    if provider:
+        stripped = _strip_model_name(model=base, custom_llm_provider=provider)
+        if stripped != base:
+            candidates.append(stripped)
+
+    for candidate in candidates:
+        entry = litellm.model_cost.get(candidate)
+        if entry is not None and entry.get("litellm_provider") is not None:
+            return dict(entry)
+    return None
+
+
 def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
     """
     Register new / Override existing models (and their pricing) to specific providers.
@@ -2933,6 +2988,26 @@ def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
             except Exception:
                 existing_model = {}
                 model_cost_key = key
+                builtin_entry = _resolve_builtin_model_cost_entry(
+                    key=_key_str, provider=provider
+                )
+                if builtin_entry is not None:
+                    for field in _CACHE_PRICING_FIELDS:
+                        if (
+                            value.get(field) is None
+                            and builtin_entry.get(field) is not None
+                        ):
+                            existing_model[field] = builtin_entry[field]
+                elif (
+                    value.get("cache_creation_input_token_cost") is None
+                    and value.get("cache_read_input_token_cost") is None
+                ):
+                    verbose_logger.warning(
+                        f"register_model: model={key} not in built-in cost map and no "
+                        "prefix/region variant matched; cache cost fields will default "
+                        "to 0. To track cache cost, add cache_creation_input_token_cost "
+                        "and cache_read_input_token_cost to model_info"
+                    )
         # ``get_model_info`` returns ``litellm_provider: None`` when the
         # provider is unknown (e.g. custom deployments registered via
         # ``Router.add_deployment``). Persisting that None into
@@ -5775,6 +5850,7 @@ def _get_model_info_helper(  # noqa: PLR0915
         ]
         split_model = potential_model_names["split_model"]
         custom_llm_provider = potential_model_names["custom_llm_provider"]
+        model_cost_custom_llm_provider = custom_llm_provider
         #########################
         provider_config: Optional[BaseLLMModelInfo] = None
         if custom_llm_provider and custom_llm_provider in LlmProvidersSet:
@@ -5840,7 +5916,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                 key = _matched_key
                 _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                 if not _check_provider_match(
-                    model_info=_model_info, custom_llm_provider=custom_llm_provider
+                    model_info=_model_info,
+                    custom_llm_provider=model_cost_custom_llm_provider,
                 ):
                     _model_info = None
             if _model_info is None:
@@ -5849,7 +5926,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5858,7 +5936,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5867,7 +5946,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5876,7 +5956,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
 
@@ -5884,7 +5965,6 @@ def _get_model_info_helper(  # noqa: PLR0915
                 raise ValueError(
                     "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
                 )
-
             _input_cost_per_token: Optional[float] = _model_info.get(
                 "input_cost_per_token"
             )
@@ -5936,6 +6016,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 cache_read_input_token_cost_above_272k_tokens=_model_info.get(
                     "cache_read_input_token_cost_above_272k_tokens", None
                 ),
+                cache_read_input_token_cost_above_512k_tokens=_model_info.get(
+                    "cache_read_input_token_cost_above_512k_tokens", None
+                ),
                 cache_read_input_token_cost_flex=_model_info.get(
                     "cache_read_input_token_cost_flex", None
                 ),
@@ -5957,6 +6040,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 input_cost_per_token_above_272k_tokens=_model_info.get(
                     "input_cost_per_token_above_272k_tokens", None
                 ),
+                input_cost_per_token_above_512k_tokens=_model_info.get(
+                    "input_cost_per_token_above_512k_tokens", None
+                ),
                 input_cost_per_query=_model_info.get("input_cost_per_query", None),
                 input_cost_per_second=_model_info.get("input_cost_per_second", None),
                 input_cost_per_audio_token=_model_info.get(
@@ -6012,6 +6098,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 output_cost_per_token_above_272k_tokens=_model_info.get(
                     "output_cost_per_token_above_272k_tokens", None
                 ),
+                output_cost_per_token_above_512k_tokens=_model_info.get(
+                    "output_cost_per_token_above_512k_tokens", None
+                ),
                 output_cost_per_second=_model_info.get("output_cost_per_second", None),
                 output_cost_per_second_1080p=_model_info.get(
                     "output_cost_per_second_1080p", None
@@ -8922,14 +9011,33 @@ class ProviderConfigManager:
         elif litellm.LlmProviders.HOSTED_VLLM == provider:
             return litellm.HostedVLLMResponsesAPIConfig()
         elif litellm.LlmProviders.BEDROCK_MANTLE == provider:
-            # Only OpenAI gpt frontier models (gpt-5.x, and future gpt-6 etc.) are
-            # served on the /openai/v1/responses path. gpt-oss and every non-OpenAI
-            # model on Mantle (nvidia, mistral, google, zai, ...) are chat-completions
-            # only and 400 on that path, so they fall through to None to keep the
-            # chat-completions emulation (see litellm/responses/main.py "config is None").
-            model_lower = model.lower() if model else ""
-            if "openai.gpt-" in model_lower and "gpt-oss" not in model_lower:
-                return litellm.BedrockMantleResponsesAPIConfig()
+            # Mantle serves Responses on two upstream paths. A model takes the
+            # /openai/v1/responses path when its price-map entry declares
+            # use_openai_responses_path (data-driven, so a non-gpt-named frontier
+            # model can be onboarded by JSON alone), or, as a fallback needing no
+            # price-map entry, when its name matches the openai.gpt- frontier
+            # convention (minus gpt-oss) -- this keeps a future gpt-6 routing
+            # correctly before its entry loads. Any other model declared
+            # mode=responses takes the standard /v1/responses path. Everything
+            # else returns None and keeps the chat-completions emulation (see
+            # responses/main.py "config is None").
+            if not model:
+                return None
+            model_lower = model.lower()
+            entry = litellm.model_cost.get(f"bedrock_mantle/{model}", {})
+            on_openai_path = entry.get("use_openai_responses_path") is True
+            name_is_frontier = (
+                "openai.gpt-" in model_lower and "gpt-oss" not in model_lower
+            )
+            if on_openai_path or name_is_frontier:
+                return litellm.BedrockMantleResponsesAPIConfig(use_openai_path=True)
+            try:
+                if get_model_info(model, "bedrock_mantle").get("mode") == "responses":
+                    return litellm.BedrockMantleResponsesAPIConfig(
+                        use_openai_path=False
+                    )
+            except Exception:
+                pass
             return None
         return None
 

model_prices_and_context_window.json

@@ -1156,6 +1156,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "azure_ai/claude-fable-5": {
+        "input_cost_per_token": 1e-05,
+        "output_cost_per_token": 5e-05,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10170,6 +10345,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10204,6 +10380,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10214,6 +10391,40 @@
         },
         "supports_output_config": true
     },
+    "claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-opus-4-8": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10238,6 +10449,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -24180,9 +24392,12 @@
         "max_output_tokens": 8192
     },
     "minimax/MiniMax-M3": {
-        "input_cost_per_token": 6e-07,
-        "output_cost_per_token": 2.4e-06,
-        "cache_read_input_token_cost": 1.2e-07,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_above_512k_tokens": 6e-07,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_512k_tokens": 2.4e-06,
+        "cache_read_input_token_cost": 6e-08,
+        "cache_read_input_token_cost_above_512k_tokens": 1.2e-07,
         "litellm_provider": "minimax",
         "mode": "chat",
         "supports_function_calling": true,
@@ -24191,7 +24406,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_vision": true,
-        "max_input_tokens": 512000,
+        "max_input_tokens": 1000000,
         "max_output_tokens": 128000
     },
     "mistral.devstral-2-123b": {
@@ -34044,6 +34259,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34072,6 +34288,67 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5@default": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34101,6 +34378,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34130,6 +34408,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -41410,6 +41689,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41429,6 +41709,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41902,5 +42183,164 @@
     "source": "https://soniox.com/pricing",
     "supported_endpoints": ["/v1/audio/transcriptions"],
     "supports_audio_input": true
+  },
+  "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 6e-07,
+    "output_cost_per_token": 3.6e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 4.5e-07,
+    "output_cost_per_token": 1.8e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/Qwen/Qwen3.6-27B-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 3.2e-07,
+    "output_cost_per_token": 3.2e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-06,
+    "output_cost_per_token": 4.4e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 202752,
+    "max_output_tokens": 202752,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/deepseek-ai/DeepSeek-V4-Flash": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-07,
+    "output_cost_per_token": 2.8e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/moonshotai/Kimi-K2.6": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 9.6e-07,
+    "output_cost_per_token": 4e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/MiniMaxAI/MiniMax-M2.5": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 3e-07,
+    "output_cost_per_token": 1.2e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 196608,
+    "max_output_tokens": 196608,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/google/gemma-4-31B-it": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-07,
+    "output_cost_per_token": 5.6e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/openai/gpt-oss-120b": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.5e-07,
+    "output_cost_per_token": 6e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 131072,
+    "max_output_tokens": 131072,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/openai/gpt-oss-20b": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 7e-08,
+    "output_cost_per_token": 2.8e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 131072,
+    "max_output_tokens": 131072,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
   }
  }

packaging/homebrew/README.md

@@ -0,0 +1,27 @@
+# Homebrew formula for the `lite` CLI
+
+[`lite.rb`](./lite.rb) is the canonical source for the Homebrew formula that installs the thin LiteLLM CLI (`litellm[cli]`). It lives here so it is versioned with the code, but Homebrew serves formulae from a tap, so it has to be published to the `BerriAI/homebrew-litellm` tap to be installable.
+
+Once published, end users install with
+
+```shell
+brew install BerriAI/litellm/lite
+```
+
+which gives them the `lite` command (`lite login`, `lite claude`, `lite models list`, ...) without the proxy server runtime. For the full proxy server, they keep using pip/uv with `litellm[proxy]` or the Docker image.
+
+## Why a tap and not homebrew-core
+
+The formula builds the published `litellm` sdist with the `cli` extra and resolves that extra's dependencies from PyPI at build time. homebrew-core forbids network access during `install` and would require every transitive dependency declared as a pinned `resource`, regenerated on each release. For a fast-moving CLI that tradeoff is not worth it, so this stays a tap formula.
+
+## Release runbook
+
+The formula can only point at a published artifact, so it activates with the first `litellm` release that ships the `cli` extra (added in [pyproject.toml](../../pyproject.toml)).
+
+1. Cut a `litellm` release whose `pyproject.toml` includes the `cli` extra and confirm it is on PyPI.
+2. Fetch the sdist URL and checksum for that version: `curl -fsSL https://pypi.org/pypi/litellm/<version>/json | jq -r '.urls[] | select(.packagetype=="sdist") | "\(.url)\n\(.digests.sha256)"'`
+3. Set `url` and `sha256` in `lite.rb` to those values; `version` is parsed from `url`.
+4. Copy `lite.rb` into the tap repo under `Formula/lite.rb`, then run `brew install --build-from-source ./Formula/lite.rb` and `brew test lite` to verify a clean build and that `lite --help` works.
+5. Commit and push to `BerriAI/homebrew-litellm`.
+
+Keep `lite.rb` here in sync with the tap copy so the in-repo formula stays the source of truth.

packaging/homebrew/lite.rb

@@ -0,0 +1,33 @@
+# Homebrew formula for the thin LiteLLM `lite` CLI (litellm[cli]).
+#
+# Ships in the BerriAI/homebrew-litellm tap, not homebrew-core: it builds the
+# published litellm sdist with the `cli` extra into a dedicated virtualenv and
+# pulls the extra's deps from PyPI. That is the low-maintenance path for a
+# fast-moving Python CLI; the resource-stanza alternative would need every
+# transitive dep re-pinned with a fresh sha256 on each release.
+#
+# RELEASE STEP (see README.md in this directory): point `url` + `sha256` at the
+# PyPI sdist of the first litellm version that ships the `cli` extra. `version`
+# is parsed from `url`, and the build installs exactly that version, so the three
+# stay in lockstep automatically.
+class Lite < Formula
+  include Language::Python::Virtualenv
+
+  desc "Thin client for the LiteLLM proxy: lite login, lite claude/codex/opencode"
+  homepage "https://docs.litellm.ai/docs/proxy/management_cli"
+  url "https://files.pythonhosted.org/packages/source/l/litellm/litellm-REPLACE_AT_RELEASE.tar.gz"
+  sha256 "REPLACE_AT_RELEASE"
+  license "MIT"
+
+  depends_on "python@3.13"
+
+  def install
+    virtualenv_create(libexec, "python3.13")
+    system libexec/"bin/pip", "install", "#{buildpath}[cli]"
+    bin.install_symlink libexec/"bin/lite"
+  end
+
+  test do
+    assert_match "login", shell_output("#{bin}/lite --help")
+  end
+end

pyproject.toml

@@ -71,6 +71,14 @@ proxy = [
     "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'",
     "pydantic-settings>=2.14.1,<3.0",
 ]
+# Thin client install for the `lite` CLI on developer laptops. The CLI's heavy
+# imports (fastapi, cryptography, ...) are all guarded, so it runs on the base
+# SDK plus just these three; none of the server runtime in `proxy` is pulled in.
+cli = [
+    "rich>=13.9.4,<14.0",
+    "pyyaml>=6.0.3,<7.0",
+    "requests>=2.32.0,<3.0",
+]
 extra_proxy = [
     "prisma>=0.11.0,<1.0",
     "azure-identity>=1.25.2,<2.0",
@@ -132,6 +140,7 @@ proxy-runtime = [
 
 [project.scripts]
 litellm = "litellm:run_server"
+lite = "litellm.proxy.client.cli:cli"
 litellm-proxy = "litellm.proxy.client.cli:cli"
 
 [dependency-groups]

scripts/install-cli.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# LiteLLM CLI Installer (the thin `lite` client)
+# Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install-cli.sh | sh
+#
+# Installs only litellm[cli]: the `lite` command for authenticating to a LiteLLM
+# proxy and running coding agents (lite claude / codex / opencode) through it.
+# None of the proxy server runtime is pulled in. To run a proxy server instead,
+# use scripts/install.sh, which installs litellm[proxy].
+#
+# Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible
+# Python itself (honouring litellm's requires-python), downloading a managed one
+# when the host has no suitable interpreter.
+#
+# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
+# ignores the shebang when invoked as `sh` and does not support `pipefail`).
+set -eu
+
+# NOTE: before merging, this must stay as "litellm[cli]" to install from PyPI.
+LITELLM_PACKAGE="litellm[cli]"
+UV_VERSION="0.10.9"
+
+# ── colours ────────────────────────────────────────────────────────────────
+if [ -t 1 ]; then
+  BOLD='\033[1m'
+  GREEN='\033[38;2;78;186;101m'
+  GREY='\033[38;2;153;153;153m'
+  RESET='\033[0m'
+else
+  BOLD='' GREEN='' GREY='' RESET=''
+fi
+
+info()    { printf "${GREY}  %s${RESET}\n" "$*"; }
+success() { printf "${GREEN}  ✔ %s${RESET}\n" "$*"; }
+header()  { printf "${BOLD}  %s${RESET}\n" "$*"; }
+die()     { printf "\n  Error: %s\n\n" "$*" >&2; exit 1; }
+
+# ── banner ─────────────────────────────────────────────────────────────────
+echo ""
+cat << 'EOF'
+  ██╗     ██╗████████╗███████╗
+  ██║     ██║╚══██╔══╝██╔════╝
+  ██║     ██║   ██║   █████╗
+  ██║     ██║   ██║   ██╔══╝
+  ███████╗██║   ██║   ███████╗
+  ╚══════╝╚═╝   ╚═╝   ╚══════╝
+EOF
+printf "  ${BOLD}LiteLLM CLI Installer${RESET}  ${GREY}the thin 'lite' client for your proxy${RESET}\n\n"
+
+# ── OS detection ───────────────────────────────────────────────────────────
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+case "$OS" in
+  Darwin)  PLATFORM="macOS ($ARCH)" ;;
+  Linux)   PLATFORM="Linux ($ARCH)" ;;
+  *)       die "Unsupported OS: $OS. LiteLLM supports macOS and Linux." ;;
+esac
+
+info "Platform: $PLATFORM"
+
+# ── uv detection / install ────────────────────────────────────────────────
+UV_BIN=""
+CURRENT_UV_VERSION=""
+for candidate in uv "$HOME/.local/bin/uv"; do
+  if command -v "$candidate" >/dev/null 2>&1; then
+    UV_BIN="$(command -v "$candidate")"
+    break
+  elif [ -x "$candidate" ]; then
+    UV_BIN="$candidate"
+    break
+  fi
+done
+
+if [ -n "$UV_BIN" ]; then
+  CURRENT_UV_VERSION="$("$UV_BIN" --version 2>/dev/null | awk '{print $2}' | head -1 || true)"
+fi
+
+if [ -z "$UV_BIN" ] || [ "${CURRENT_UV_VERSION:-}" != "$UV_VERSION" ]; then
+  header "Installing uv…"
+  if [ -n "${CURRENT_UV_VERSION:-}" ]; then
+    info "Upgrading uv from ${CURRENT_UV_VERSION} to ${UV_VERSION}"
+  fi
+  curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_NO_MODIFY_PATH=1 sh \
+    || die "uv installation failed. Try manually: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh"
+  UV_BIN="$HOME/.local/bin/uv"
+fi
+
+# ── install ────────────────────────────────────────────────────────────────
+# --python-preference system: reuse a compatible system Python when present,
+# otherwise download a managed one. Either way uv honours litellm's requires-python,
+# so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced.
+echo ""
+header "Installing litellm[cli]…"
+echo ""
+
+"$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \
+  || die "uv tool install failed. Try manually: $UV_BIN tool install '${LITELLM_PACKAGE}'"
+
+# ── find the lite binary installed by uv tool ──────────────────────────────
+SCRIPTS_DIR="$("$UV_BIN" tool dir --bin)"
+LITE_BIN="${SCRIPTS_DIR}/lite"
+
+if [ ! -x "$LITE_BIN" ]; then
+  die "lite binary not found after install. Try: $UV_BIN tool install '${LITELLM_PACKAGE}'"
+fi
+
+# ── success banner ─────────────────────────────────────────────────────────
+echo ""
+success "LiteLLM CLI installed"
+
+installed_ver="$("$LITE_BIN" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
+[ -n "$installed_ver" ] && info "Version: $installed_ver"
+
+# ── PATH hint ──────────────────────────────────────────────────────────────
+if ! command -v lite >/dev/null 2>&1; then
+  info "Note: add lite to your PATH:  export PATH=\"\$PATH:${SCRIPTS_DIR}\""
+fi
+
+# ── next steps ─────────────────────────────────────────────────────────────
+echo ""
+header "Next steps:"
+echo ""
+info "  export LITELLM_PROXY_URL=https://your-proxy   # point at your gateway"
+info "  lite login                                    # authenticate via SSO"
+info "  lite claude                                   # run Claude Code through the proxy"
+echo ""
+info "Docs: https://docs.litellm.ai/docs/proxy/management_cli"
+echo ""

scripts/install.sh

@@ -2,13 +2,13 @@
 # LiteLLM Installer
 # Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh
 #
+# Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible
+# Python itself (reusing a suitable system one, else downloading a managed build).
+#
 # NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
 # ignores the shebang when invoked as `sh` and does not support `pipefail`).
 set -eu
 
-MIN_PYTHON_MAJOR=3
-MIN_PYTHON_MINOR=9
-
 # NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI.
 LITELLM_PACKAGE="litellm[proxy]"
 UV_VERSION="0.10.9"
@@ -52,27 +52,6 @@ esac
 
 info "Platform: $PLATFORM"
 
-# ── Python detection ───────────────────────────────────────────────────────
-PYTHON_BIN=""
-for candidate in python3 python; do
-  if command -v "$candidate" >/dev/null 2>&1; then
-    major="$("$candidate" -c 'import sys; print(sys.version_info.major)' 2>/dev/null || true)"
-    minor="$("$candidate" -c 'import sys; print(sys.version_info.minor)' 2>/dev/null || true)"
-    if [ "${major:-0}" -ge "$MIN_PYTHON_MAJOR" ] && [ "${minor:-0}" -ge "$MIN_PYTHON_MINOR" ]; then
-      PYTHON_BIN="$(command -v "$candidate")"
-      info "Python: $("$candidate" --version 2>&1)"
-      break
-    fi
-  fi
-done
-
-if [ -z "$PYTHON_BIN" ]; then
-  die "Python ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but not found.
-  Install it from https://python.org/downloads or via your package manager:
-    macOS:  brew install python@3
-    Ubuntu: sudo apt install python3"
-fi
-
 # ── uv detection / install ────────────────────────────────────────────────
 UV_BIN=""
 CURRENT_UV_VERSION=""
@@ -105,15 +84,18 @@ echo ""
 header "Installing litellm[proxy]…"
 echo ""
 
-"$UV_BIN" tool install --python "$PYTHON_BIN" --force "${LITELLM_PACKAGE}" \
-  || die "uv tool install failed. Try manually: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'"
+# --python-preference system: reuse a compatible system Python when present,
+# otherwise download a managed one. Either way uv honours litellm's requires-python,
+# so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced.
+"$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \
+  || die "uv tool install failed. Try manually: $UV_BIN tool install '${LITELLM_PACKAGE}'"
 
 # ── find the litellm binary installed by uv tool ───────────────────────────
 SCRIPTS_DIR="$("$UV_BIN" tool dir --bin)"
 LITELLM_BIN="${SCRIPTS_DIR}/litellm"
 
 if [ ! -x "$LITELLM_BIN" ]; then
-  die "litellm binary not found after install. Try: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'"
+  die "litellm binary not found after install. Try: $UV_BIN tool install '${LITELLM_PACKAGE}'"
 fi
 
 # ── success banner ─────────────────────────────────────────────────────────

security.md

@@ -3,12 +3,20 @@
 
 ## Security Vulnerability Reporting Guidelines
 
+> [!WARNING]
+> Reports that do not include a video demonstrating the exploit will be closed without review. See [Reproduction Video Requirement](#reproduction-video-requirement) below.
+
 We value the security community's role in protecting our systems and users. To report a security vulnerability:
 
 - File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new)
 - Include steps to reproduce the issue
+- Include a video or screen recording demonstrating the full exploit against a live LiteLLM instance, from initial access through to impact. A terminal recording (for example asciinema) is fine for CLI-only exploits.
 - Provide any relevant additional information
 
+### Reproduction Video Requirement
+
+A video demonstrating the exploit is required for every report. AI tools have made it easy to produce plausible-sounding vulnerability reports that do not reproduce in practice, and triaging them takes time away from real issues. Reports submitted without a working reproduction video will be closed without review. If you add a video to a closed report, we will reopen and triage it.
+
 ### Vulnerability Categories
 
 We classify vulnerabilities into the following categories:
@@ -38,7 +46,7 @@ We offer bounties for responsibly disclosed vulnerabilities based on severity:
 | **Medium** | N/A | P2 authenticated privilege escalation |
 | **Low** | N/A | Minor information disclosure, low-impact misconfigurations |
 
-To qualify for a bounty, reports must include clear reproduction steps and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days.
+To qualify for a bounty, reports must include clear reproduction steps, a reproduction video as described above, and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days.
 
 ### Known Non-Issues
 

tests/llm_translation/reasoning_effort_grid/grid_spec.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 from typing import Dict, FrozenSet, List, Optional, Tuple
 
-
 OMIT = object()
 
 
@@ -136,6 +135,13 @@ _CAPS_NONE: FrozenSet[str] = frozenset()
 
 
 ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="claude-fable-5",
+        model="anthropic/claude-fable-5",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_XHIGH_MAX,
+    ),
     ModelEntry(
         alias="claude-opus-4-8",
         model="anthropic/claude-opus-4-8",
@@ -168,6 +174,19 @@ ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
 
 
 AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="azure-claude-fable-5",
+        model="azure_ai/claude-fable-5",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        fail_reason=(
+            "claude-fable-5 has no deployment on the CI Microsoft Foundry "
+            "resource yet; Foundry returns DeploymentNotFound until someone "
+            "creates the fable-5 deployment, so this cell stays loud in CI. "
+            "Remove this fail_reason once the deployment exists."
+        ),
+    ),
     ModelEntry(
         alias="azure-claude-opus-4-8",
         model="azure_ai/claude-opus-4-8",
@@ -213,6 +232,20 @@ AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
 
 
 VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="vertex-claude-fable-5",
+        model="vertex_ai/claude-fable-5",
+        mode="adaptive",
+        extra_params=(("vertex_location", "global"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        fail_reason=(
+            "claude-fable-5 availability on the CI Vertex project is not yet "
+            "confirmed for this brand-new release, so this cell stays loud in "
+            "CI until verified. Remove this fail_reason once the model is "
+            "confirmed available on the global Vertex endpoint."
+        ),
+    ),
     ModelEntry(
         alias="vertex-claude-opus-4-8",
         model="vertex_ai/claude-opus-4-8",
@@ -263,6 +296,23 @@ VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
 
 
 BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-claude-fable-5",
+        model="bedrock/converse/us.anthropic.claude-fable-5",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        bedrock_effort_ceiling="xhigh",
+        unavailable_error="is not available for this account",
+        fail_reason=(
+            "claude-fable-5 on Bedrock requires the account to opt in to "
+            "provider data sharing (data retention mode "
+            "'provider_data_sharing' via the Data Retention API); the CI "
+            "account has not opted in yet, so this cell stays loud in CI. "
+            "Remove this fail_reason once the opt-in is done."
+        ),
+    ),
     ModelEntry(
         alias="bedrock-claude-opus-4-8",
         model="bedrock/converse/us.anthropic.claude-opus-4-8",

tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py

@@ -15,7 +15,6 @@ from .grid_spec import (
     all_cells,
 )
 
-
 _PROMPT_MESSAGES: List[Dict[str, str]] = [
     {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
 ]
@@ -201,8 +200,8 @@ async def test_reasoning_effort_grid(
 
 
 def test_grid_cell_count() -> None:
-    assert len(_PARAMS) == 25 * 11, (
-        f"expected 275 cells (25 provider x model combos x 11 efforts), "
+    assert len(_PARAMS) == 29 * 11, (
+        f"expected 319 cells (29 provider x model combos x 11 efforts), "
         f"got {len(_PARAMS)}"
     )
 

tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py

@@ -2425,6 +2425,42 @@ class TestConvertToModelResponseObjectCompletion:
         assert result.choices[0].message.content == "The answer is 4."
         assert result.choices[0].message.reasoning_content == "2+2=4"
 
+    def test_reasoning_content_not_mirrored_into_provider_specific_fields(self):
+        """Mirroring reasoning_content into provider_specific_fields made
+        cache-replayed messages diverge from live Anthropic messages, which
+        only set it top-level, breaking cache key stability (issue #27337)."""
+        response_object = {
+            "id": "chatcmpl-5",
+            "model": "claude-sonnet-4-5",
+            "choices": [
+                {
+                    "finish_reason": "stop",
+                    "index": 0,
+                    "message": {
+                        "content": "The answer is 4.",
+                        "role": "assistant",
+                        "reasoning_content": "2+2=4",
+                        "thinking_blocks": [
+                            {
+                                "type": "thinking",
+                                "thinking": "2+2=4",
+                                "signature": "sig",
+                            }
+                        ],
+                    },
+                }
+            ],
+            "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15},
+        }
+
+        result = convert_to_model_response_object(
+            response_object=response_object,
+            model_response_object=ModelResponse(),
+        )
+        message = result.choices[0].message
+        assert message.reasoning_content == "2+2=4"
+        assert "reasoning_content" not in (message.provider_specific_fields or {})
+
     def test_response_none_raises(self):
         with pytest.raises(Exception):
             convert_to_model_response_object(

tests/local_testing/test_basic_python_version.py

@@ -92,6 +92,56 @@ def test_package_dependencies():
         )
 
 
+def test_cli_extra_is_a_thin_client_install():
+    """The `cli` extra must install a working `lite` client without dragging in the
+    proxy server runtime. It therefore has to declare the CLI's real third-party
+    deps (rich, pyyaml, requests) and must never contain a server-only dependency
+    from the `proxy` extra; a leak there silently re-bloats the laptop install.
+    """
+    import pathlib
+
+    import litellm
+    from packaging.requirements import Requirement
+
+    try:
+        import tomllib as tomli
+    except ImportError:
+        try:
+            import tomli
+        except ImportError:
+            pytest.skip("tomli/tomllib not available - skipping dependency check")
+
+    pyproject_path = pathlib.Path(litellm.__file__).parent.parent / "pyproject.toml"
+    with open(pyproject_path, "rb") as f:
+        optional_deps = tomli.load(f)["project"]["optional-dependencies"]
+
+    assert "cli" in optional_deps, "Expected a `cli` extra for the thin lite install"
+
+    cli_names = {Requirement(req).name.lower() for req in optional_deps["cli"]}
+
+    missing = {"rich", "pyyaml", "requests"} - cli_names
+    assert not missing, f"`cli` extra is missing deps the lite CLI imports: {missing}"
+
+    server_only = {
+        "fastapi",
+        "uvicorn",
+        "gunicorn",
+        "granian",
+        "starlette",
+        "boto3",
+        "polars",
+        "soundfile",
+        "mcp",
+        "cryptography",
+        "apscheduler",
+        "rq",
+        "litellm-enterprise",
+        "litellm-proxy-extras",
+    }
+    leaked = cli_names & server_only
+    assert not leaked, f"`cli` extra leaks proxy-server deps onto laptops: {leaked}"
+
+
 import os
 import subprocess
 import time

tests/local_testing/test_router_debug_logs.py

@@ -82,7 +82,9 @@ def test_async_fallbacks(caplog):
     asyncio.run(_make_request())
     captured_logs = [rec.message for rec in caplog.records]
 
-    # on circle ci the captured logs get some async task exception logs - filter them out "Task exception was never retrieved"
+    # on circle ci the captured logs get async cleanup noise from the gc (leaked
+    # task warnings, plus aiohttp "Unclosed client session"/"Unclosed connector"
+    # warnings from cached clients other router tests evicted) - filter it out
     captured_logs = [
         log
         for log in captured_logs
@@ -90,6 +92,8 @@ def test_async_fallbacks(caplog):
         and "Task was destroyed but it is pending" not in log
         and "get_available_deployment" not in log
         and "in the Langfuse queue" not in log
+        and "Unclosed client session" not in log
+        and "Unclosed connector" not in log
     ]
 
     print("\n Captured caplog records - ", captured_logs)

tests/proxy_behavior/management/test_team_budget_limits.py

@@ -28,7 +28,7 @@ import pytest
 from litellm.proxy.utils import hash_token
 
 from .actors import Actor
-from .conftest import create_scratch_org, create_scratch_team
+from .conftest import MASTER_KEY, create_scratch_org, create_scratch_team
 
 pytestmark = pytest.mark.asyncio(loop_scope="session")
 
@@ -288,34 +288,130 @@ async def test_check_user_team_limits(
 
 
 # ---------------------------------------------------------------------------
-# /team/update path — _check_user_team_limits on existing team, no-org.
-# Pin one over-budget rejection here so the update-side wiring is also
-# covered (the update path is a second call site with its own data shape).
+# /team/update path — budget authority.
+#
+# The caller's PERSONAL limits are never applied on update (that compared the
+# wrong thing). But raising a team's spend ceiling is reserved for proxy admins:
+# a team admin may keep or LOWER the budget, only a proxy admin may RAISE it.
+# _check_user_team_limits() only runs on /team/new.
 # ---------------------------------------------------------------------------
 
 
-async def test_team_update_user_limit_rejected(proxy_client, prisma, scratch):
+async def test_team_admin_raise_budget_blocked(proxy_client, prisma, scratch):
+    """A team admin cannot raise the team's budget; the block is NOT based on
+    their personal budget (which here is higher than the requested value)."""
     caller_cleartext = await _seed_scratch_actor_with_caps(
         prisma,
         scratch.prefix,
-        max_budget=100.0,
+        max_budget=100000.0,  # generous personal budget; must not matter
     )
     creator_user_id = f"{scratch.prefix}-team-creator"
-    # Team must exist before /team/update; seed a standalone scratch team
-    # owned by the same actor so the update authz gate passes.
     team_id = await create_scratch_team(
         prisma,
         team_id=scratch.tag("team"),
         admin_user_ids=[creator_user_id],
         max_budget=50.0,
     )
+    # Raise the team budget 50 -> 999 as a team admin.
     resp = await proxy_client.post(
         "/team/update",
         headers={"Authorization": f"Bearer {caller_cleartext}"},
         json={"team_id": team_id, "max_budget": 999.0},
     )
-    assert resp.status_code == 400, resp.text
+    assert resp.status_code == 403, resp.text
 
     row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id})
     assert row is not None
-    assert row.max_budget == 50.0, "row max_budget mutated despite rejection"
+    assert row.max_budget == 50.0, "team budget must not change on a blocked raise"
+
+
+async def test_team_admin_lower_budget_allowed(proxy_client, prisma, scratch):
+    """A team admin may freely lower (or keep) the team's budget."""
+    caller_cleartext = await _seed_scratch_actor_with_caps(
+        prisma,
+        scratch.prefix,
+        max_budget=10.0,  # below both the old and new team budget; must not matter
+    )
+    creator_user_id = f"{scratch.prefix}-team-creator"
+    team_id = await create_scratch_team(
+        prisma,
+        team_id=scratch.tag("team"),
+        admin_user_ids=[creator_user_id],
+        max_budget=500.0,
+    )
+    # Lower the team budget 500 -> 300 as a team admin.
+    resp = await proxy_client.post(
+        "/team/update",
+        headers={"Authorization": f"Bearer {caller_cleartext}"},
+        json={"team_id": team_id, "max_budget": 300.0},
+    )
+    assert resp.status_code == 200, resp.text
+
+    row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    assert row is not None
+    assert row.max_budget == 300.0, "team admin should be able to lower the budget"
+
+
+async def test_proxy_admin_raise_budget_allowed(proxy_client, prisma, scratch):
+    """A proxy admin may raise a team's budget."""
+    team_id = await create_scratch_team(
+        prisma,
+        team_id=scratch.tag("team"),
+        admin_user_ids=[f"{scratch.prefix}-team-creator"],
+        max_budget=50.0,
+    )
+    # MASTER_KEY acts as proxy admin.
+    resp = await proxy_client.post(
+        "/team/update",
+        headers={"Authorization": f"Bearer {MASTER_KEY}"},
+        json={"team_id": team_id, "max_budget": 999.0},
+    )
+    assert resp.status_code == 200, resp.text
+
+    row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    assert row is not None
+    assert row.max_budget == 999.0, "proxy admin should be able to raise the budget"
+
+
+async def test_team_admin_remove_budget_cap_blocked(proxy_client, prisma, scratch):
+    """A team admin cannot strip the team's cap (max_budget=null); removing the
+    ceiling is the strongest possible raise -> proxy-admin only."""
+    caller_cleartext = await _seed_scratch_actor_with_caps(
+        prisma, scratch.prefix, max_budget=100000.0
+    )
+    team_id = await create_scratch_team(
+        prisma,
+        team_id=scratch.tag("team"),
+        admin_user_ids=[f"{scratch.prefix}-team-creator"],
+        max_budget=50.0,
+    )
+    resp = await proxy_client.post(
+        "/team/update",
+        headers={"Authorization": f"Bearer {caller_cleartext}"},
+        json={"team_id": team_id, "max_budget": None},
+    )
+    assert resp.status_code == 403, resp.text
+
+    row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    assert row is not None
+    assert row.max_budget == 50.0, "team budget cap must not be removed by a team admin"
+
+
+async def test_proxy_admin_remove_budget_cap_allowed(proxy_client, prisma, scratch):
+    """A proxy admin may remove a team's cap (max_budget=null)."""
+    team_id = await create_scratch_team(
+        prisma,
+        team_id=scratch.tag("team"),
+        admin_user_ids=[f"{scratch.prefix}-team-creator"],
+        max_budget=50.0,
+    )
+    resp = await proxy_client.post(
+        "/team/update",
+        headers={"Authorization": f"Bearer {MASTER_KEY}"},
+        json={"team_id": team_id, "max_budget": None},
+    )
+    assert resp.status_code == 200, resp.text
+
+    row = await prisma.db.litellm_teamtable.find_unique(where={"team_id": team_id})
+    assert row is not None
+    assert row.max_budget is None, "proxy admin should be able to remove the cap"

tests/router_unit_tests/test_router_endpoints.py

@@ -1236,3 +1236,60 @@ async def test_init_containers_api_endpoints_managed_id_without_model_id_applies
     assert call_kw["container_id"] == "cfile_upstream_abc"
     assert call_kw["file_id"] == "cfile_xyz"
     assert call_kw["custom_llm_provider"] == "azure"
+
+
+def test_router_model_group_encrypted_content_affinity_callback_registration():
+    from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
+        DeploymentAffinityCheck,
+    )
+    from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
+        EncryptedContentAffinityCheck,
+    )
+
+    model_group = "openai.gpt-5.1-codex"
+    model_group_affinity_config = {
+        model_group: ["encrypted_content_affinity"],
+    }
+    router = Router(
+        model_list=[
+            {
+                "model_name": model_group,
+                "litellm_params": {
+                    "model": "openai/gpt-5.1-codex",
+                    "api_key": "mock-api-key",
+                },
+            }
+        ],
+        model_group_affinity_config=model_group_affinity_config,
+        num_retries=0,
+    )
+
+    try:
+        callbacks = router.optional_callbacks or []
+        encrypted_content_callbacks = [
+            cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck)
+        ]
+        deployment_callback = next(
+            cb for cb in callbacks if isinstance(cb, DeploymentAffinityCheck)
+        )
+        assert len(encrypted_content_callbacks) == 1
+        assert encrypted_content_callbacks[0].enable_global_affinity is False
+        assert (
+            encrypted_content_callbacks[0].model_group_affinity_config
+            == model_group_affinity_config
+        )
+        assert callbacks.index(encrypted_content_callbacks[0]) < callbacks.index(
+            deployment_callback
+        )
+
+        router._add_encrypted_content_affinity_check(enable_global_affinity=True)
+
+        callbacks = router.optional_callbacks or []
+        encrypted_content_callbacks = [
+            cb for cb in callbacks if isinstance(cb, EncryptedContentAffinityCheck)
+        ]
+        assert len(encrypted_content_callbacks) == 1
+        assert encrypted_content_callbacks[0].enable_global_affinity is True
+        assert encrypted_content_callbacks[0].router is router
+    finally:
+        router.discard()

tests/test_litellm/caching/test_caching.py

@@ -3,6 +3,7 @@ import re
 
 from litellm.caching.caching import Cache
 from litellm.types.caching import LiteLLMCacheType
+from litellm.types.utils import Embedding, EmbeddingResponse, Usage
 
 
 def test_cache_key_debug_log_does_not_include_prompt_material(caplog):
@@ -41,8 +42,37 @@ def test_cache_key_debug_log_does_not_include_prompt_material(caplog):
     assert re.fullmatch(r"[0-9a-f]{64}", cache_key)
 
     created_cache_key_logs = [
-        record.getMessage() for record in caplog.records if "Created cache key:" in record.getMessage()
+        record.getMessage()
+        for record in caplog.records
+        if "Created cache key:" in record.getMessage()
     ]
     assert created_cache_key_logs
     assert all(prompt_marker not in message for message in created_cache_key_logs)
     assert any(cache_key in message for message in created_cache_key_logs)
+
+
+def _embedding_response(prompt_tokens, num_items):
+    return EmbeddingResponse(
+        model="amazon.titan-embed-image-v1",
+        data=[
+            Embedding(embedding=[0.0], index=i, object="embedding")
+            for i in range(num_items)
+        ],
+        usage=Usage(
+            prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens
+        ),
+    )
+
+
+def test_get_per_item_prompt_tokens_single_item_returns_full_value():
+    cache = Cache(type=LiteLLMCacheType.LOCAL)
+    result = _embedding_response(prompt_tokens=0, num_items=1)
+    assert cache._get_per_item_prompt_tokens(result, 0) == 0
+
+
+def test_get_per_item_prompt_tokens_distributes_with_remainder():
+    cache = Cache(type=LiteLLMCacheType.LOCAL)
+    result = _embedding_response(prompt_tokens=10, num_items=3)
+    per_item = [cache._get_per_item_prompt_tokens(result, i) for i in range(3)]
+    assert sum(per_item) == 10  # 4 + 3 + 3
+    assert per_item == [4, 3, 3]

tests/test_litellm/caching/test_caching_handler.py

@@ -436,3 +436,123 @@ def test_convert_cached_responses_legacy_stream_path():
     )
 
     assert isinstance(result, CachedResponsesAPIStreamingIterator)
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_restores_stored_prompt_tokens_for_image_input():
+    """Image-embedding cache hit restores prompt_tokens=0 from the stored value
+    instead of recomputing a bogus count by tokenizing the base64 input."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    # base64-like blob — token_counter over this would return a large nonzero count
+    image_input = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk" * 50
+
+    cached_result = [
+        {
+            "embedding": [-0.025, -0.019],
+            "index": 0,
+            "object": "embedding",
+            "model": "amazon.titan-embed-image-v1",
+            "prompt_tokens": 0,
+            "prompt_tokens_details": {"image_count": 1},
+        }
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={"model": "amazon.titan-embed-image-v1", "input": image_input},
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="amazon.titan-embed-image-v1",
+    )
+
+    assert cache_hit
+    assert response.usage is not None
+    assert response.usage.prompt_tokens == 0
+    assert response.usage.total_tokens == 0
+    assert response.usage.prompt_tokens_details.image_count == 1
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_sums_stored_prompt_tokens_across_items():
+    """A multi-item cache hit sums the stored per-item prompt_tokens back to the total."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    cached_result = [
+        {
+            "embedding": [-0.01],
+            "index": 0,
+            "object": "embedding",
+            "model": "text-embedding-3-small",
+            "prompt_tokens": 5,
+        },
+        {
+            "embedding": [-0.02],
+            "index": 1,
+            "object": "embedding",
+            "model": "text-embedding-3-small",
+            "prompt_tokens": 4,
+        },
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={"model": "text-embedding-3-small", "input": ["hello world", "foo bar"]},
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="text-embedding-3-small",
+    )
+
+    assert cache_hit
+    assert response.usage.prompt_tokens == 9
+    assert response.usage.total_tokens == 9
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_falls_back_to_token_counter_for_legacy_entries():
+    """Legacy cache entries with no stored prompt_tokens still recompute via token_counter
+    for str inputs (backward compatibility)."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    # No prompt_tokens key — pre-fix entry
+    cached_result = [
+        {
+            "embedding": [-0.025, -0.019],
+            "index": 0,
+            "object": "embedding",
+            "model": "text-embedding-ada-002",
+        },
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={"model": "text-embedding-ada-002", "input": "hello world"},
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="text-embedding-ada-002",
+    )
+
+    assert cache_hit
+    # token_counter over "hello world" yields a nonzero count — fallback path still runs
+    assert response.usage.prompt_tokens > 0

tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py

@@ -328,6 +328,41 @@ def test_generic_cost_per_token_gpt54_above_272k_tokens():
     assert round(completion_cost, 10) == round(expected_completion, 10)
 
 
+def test_generic_cost_per_token_minimax_m3_above_512k_tokens():
+    """MiniMax-M3: prompts >512K input tokens priced at 2x input, output, and cache read."""
+    model = "minimax/MiniMax-M3"
+    custom_llm_provider = "minimax"
+    os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+
+    model_cost_map = litellm.model_cost[model]
+    prompt_tokens = 600000
+    cached_tokens = 100000
+    completion_tokens = 1000
+    usage = Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        total_tokens=prompt_tokens + completion_tokens,
+        prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=cached_tokens),
+    )
+    prompt_cost, completion_cost = generic_cost_per_token(
+        model=model,
+        usage=usage,
+        custom_llm_provider=custom_llm_provider,
+    )
+    expected_prompt = (
+        model_cost_map["input_cost_per_token_above_512k_tokens"]
+        * (prompt_tokens - cached_tokens)
+        + model_cost_map["cache_read_input_token_cost_above_512k_tokens"]
+        * cached_tokens
+    )
+    expected_completion = (
+        model_cost_map["output_cost_per_token_above_512k_tokens"] * completion_tokens
+    )
+    assert round(prompt_cost, 10) == round(expected_prompt, 10)
+    assert round(completion_cost, 10) == round(expected_completion, 10)
+
+
 def test_generic_cost_per_token_gpt55():
     """gpt-5.5: base pricing — $5/1M input, $30/1M output, $0.50/1M cached input."""
     model = "gpt-5.5"

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -11,6 +11,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     BedrockImageProcessor,
     _bedrock_converse_messages_pt,
     _bedrock_tools_pt,
+    _rename_duplicate_bedrock_document_names,
     _convert_to_bedrock_tool_call_invoke,
     _convert_to_bedrock_tool_call_result,
     anthropic_messages_pt,
@@ -2809,6 +2810,93 @@ def test_bedrock_converse_messages_pt_document_deterministic_name():
     assert name1 == name2
 
 
+def test_bedrock_converse_messages_pt_renames_duplicate_document_names():
+    """
+    The same document in multiple turns must not produce duplicate names;
+    Bedrock rejects requests with "Messages can not contain duplicate
+    document names". The first occurrence keeps its hash-based name and
+    later occurrences get a deterministic positional suffix.
+    """
+    document_block = {
+        "type": "document",
+        "source": {
+            "type": "base64",
+            "media_type": "application/pdf",
+            "data": "dGVzdA==",
+        },
+    }
+    messages = [
+        {
+            "role": "user",
+            "content": [document_block, {"type": "text", "text": "summarize this"}],
+        },
+        {"role": "assistant", "content": "It says test."},
+        {
+            "role": "user",
+            "content": [document_block, {"type": "text", "text": "summarize again"}],
+        },
+    ]
+
+    result1 = _bedrock_converse_messages_pt(
+        messages, "anthropic.claude-sonnet-4-6", "bedrock"
+    )
+    result2 = _bedrock_converse_messages_pt(
+        messages, "anthropic.claude-sonnet-4-6", "bedrock"
+    )
+
+    names1 = [
+        block["document"]["name"]
+        for message in result1
+        for block in message["content"]
+        if "document" in block
+    ]
+    names2 = [
+        block["document"]["name"]
+        for message in result2
+        for block in message["content"]
+        if "document" in block
+    ]
+
+    assert len(names1) == 2
+    assert len(set(names1)) == 2
+    assert names1[1] == f"{names1[0]}_2"
+    assert names1 == names2
+
+    single_turn = _bedrock_converse_messages_pt(
+        [messages[0]], "anthropic.claude-sonnet-4-6", "bedrock"
+    )
+    assert names1[0] == single_turn[0]["content"][0]["document"]["name"]
+
+
+def test_rename_duplicate_bedrock_document_names_skips_organic_suffixes():
+    """
+    A renamed duplicate must not collide with a document whose organic name
+    already carries the would-be suffix (e.g. an existing ``report_2``),
+    regardless of whether that document appears before or after the rename.
+    """
+
+    def _contents(names):
+        return [
+            {
+                "role": "user",
+                "content": [{"document": {"name": name}} for name in names],
+            }
+        ]
+
+    def _names(contents):
+        return [block["document"]["name"] for block in contents[0]["content"]]
+
+    organic_first = _rename_duplicate_bedrock_document_names(
+        _contents(["report", "report_2", "report"])
+    )
+    assert _names(organic_first) == ["report", "report_2", "report_3"]
+
+    organic_last = _rename_duplicate_bedrock_document_names(
+        _contents(["report", "report", "report_2"])
+    )
+    assert _names(organic_last) == ["report", "report_3", "report_2"]
+
+
 def test_bedrock_converse_messages_pt_document_rejects_url_source():
     """Test that a URL-type document source raises a clear error instead of KeyError."""
     messages = [

tests/test_litellm/litellm_core_utils/test_xai_oauth_routing.py

@@ -0,0 +1,81 @@
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath("../../.."))
+
+import litellm
+from litellm import LlmProviders
+from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
+from litellm.litellm_core_utils.get_llm_provider_logic import (
+    _get_openai_compatible_provider_info,
+)
+from litellm.llms.xai.chat.transformation import XAIChatConfig
+from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig
+from litellm.types.router import GenericLiteLLMParams
+from litellm.utils import (
+    ProviderConfigManager,
+    get_optional_params,
+    validate_environment,
+)
+
+
+def test_xai_provider_config_routing():
+    chat_config = ProviderConfigManager.get_provider_chat_config(
+        model="grok-3-mini",
+        provider=LlmProviders.XAI,
+    )
+    responses_config = ProviderConfigManager.get_provider_responses_api_config(
+        model="grok-3-mini",
+        provider=LlmProviders.XAI,
+    )
+
+    assert isinstance(chat_config, XAIChatConfig)
+    assert isinstance(responses_config, XAIResponsesAPIConfig)
+
+
+def test_xai_openai_compatible_provider_info():
+    model, custom_llm_provider, dynamic_api_key, api_base = (
+        _get_openai_compatible_provider_info(
+            model="xai/grok-3-mini",
+            api_base="https://api.x.ai/v1",
+            api_key="api-key",
+            dynamic_api_key=None,
+        )
+    )
+
+    assert model == "grok-3-mini"
+    assert custom_llm_provider == "xai"
+    assert api_base == "https://api.x.ai/v1"
+    assert dynamic_api_key == "api-key"
+
+
+def test_xai_get_model_info_uses_xai_pricing_metadata():
+    model_info = litellm.get_model_info("xai/grok-3-mini")
+
+    assert model_info["litellm_provider"] == "xai"
+    assert model_info["key"] == "xai/grok-3-mini"
+    assert model_info["mode"] == "chat"
+
+
+def test_xai_validate_environment_reads_api_key(monkeypatch):
+    monkeypatch.setenv("XAI_API_KEY", "api-key")
+
+    result = validate_environment(model="xai/grok-3-mini")
+
+    assert result == {"keys_in_environment": True, "missing_keys": []}
+
+
+def test_xai_oauth_flag_is_generic_litellm_param():
+    litellm_params = GenericLiteLLMParams(use_xai_oauth=True)
+    runtime_params = get_litellm_params(use_xai_oauth=True)
+    result = get_optional_params(
+        model="grok-3-mini",
+        custom_llm_provider="xai",
+        temperature=0.2,
+        drop_params=True,
+    )
+
+    assert result["temperature"] == 0.2
+    assert litellm_params.use_xai_oauth is True
+    assert runtime_params["use_xai_oauth"] is True
+    assert "use_xai_oauth" not in result

tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py

@@ -5261,6 +5261,8 @@ def test_should_strip_billing_metadata_by_provider(
 
     config_cls = getattr(importlib.import_module(module_path), class_name)
     assert config_cls().should_strip_billing_metadata() is expected_strip
+
+
 def test_namespace_tool_flat_nested_tools_are_extracted():
     """Codex sends nested tools in flat format {type, name, description, parameters} with no 'function' wrapper.
     These must be normalized and mapped without raising KeyError: 'function'."""
@@ -5357,3 +5359,140 @@ def test_client_metadata_stripped_from_anthropic_request():
         headers={},
     )
     assert "client_metadata" not in result
+
+
+@pytest.mark.parametrize(
+    "model",
+    ["claude-fable-5", "claude-opus-4-7", "claude-opus-4-8-20260120"],
+)
+def test_sampling_params_dropped_for_models_that_removed_them(model):
+    """Fable 5 / Opus 4.7 / 4.8 reject temperature != 1 and any top_p with a
+    400; with drop_params set they must be dropped, not forwarded (#30064)."""
+    config = AnthropicConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"temperature": 0.5, "top_p": 0.9},
+        optional_params={},
+        model=model,
+        drop_params=True,
+    )
+
+    assert "temperature" not in result
+    assert "top_p" not in result
+
+
+@pytest.mark.parametrize("params", [{"temperature": 0.5}, {"top_p": 0.9}, {"top_p": 1}])
+def test_sampling_params_raise_clean_error_without_drop_params(params, monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", False)
+    config = AnthropicConfig()
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
+        config.map_openai_params(
+            non_default_params=params,
+            optional_params={},
+            model="claude-fable-5",
+            drop_params=False,
+        )
+
+
+def test_temperature_1_forwarded_on_models_that_removed_sampling_params():
+    """temperature=1 (the API default) is still accepted and must pass through."""
+    config = AnthropicConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"temperature": 1},
+        optional_params={},
+        model="claude-fable-5",
+        drop_params=False,
+    )
+
+    assert result["temperature"] == 1
+
+
+@pytest.mark.parametrize("model", ["claude-opus-4-6", "claude-sonnet-4-6"])
+def test_sampling_params_forwarded_on_models_that_accept_them(model):
+    config = AnthropicConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"temperature": 0.5, "top_p": 0.9},
+        optional_params={},
+        model=model,
+        drop_params=True,
+    )
+
+    assert result["temperature"] == 0.5
+    assert result["top_p"] == 0.9
+
+
+def test_sampling_param_gating_driven_by_model_map_flag(monkeypatch):
+    """The drop/raise decision must come from ``supports_sampling_params`` in
+    the model map, not just name matching: a flagged entry gates a model whose
+    name says nothing, and an explicit ``true`` overrides the name fallback."""
+    monkeypatch.setitem(
+        litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}
+    )
+    monkeypatch.setitem(
+        litellm.model_cost, "claude-fable-5-test", {"supports_sampling_params": True}
+    )
+    config = AnthropicConfig()
+
+    flagged_off = config.map_openai_params(
+        non_default_params={"top_p": 0.9},
+        optional_params={},
+        model="claude-zeta-9",
+        drop_params=True,
+    )
+    assert "top_p" not in flagged_off
+
+    flagged_on = config.map_openai_params(
+        non_default_params={"top_p": 0.9},
+        optional_params={},
+        model="claude-fable-5-test",
+        drop_params=True,
+    )
+    assert flagged_on["top_p"] == 0.9
+
+
+def test_top_k_dropped_at_transform_for_models_that_removed_it():
+    """``top_k`` is a provider-specific kwarg that bypasses
+    ``map_openai_params``, so it must be stripped at the transform_request
+    boundary shared by the direct, invoke, Vertex, and Azure paths (#30064)."""
+    config = AnthropicConfig()
+
+    result = config.transform_request(
+        model="claude-fable-5",
+        messages=[{"role": "user", "content": "hello"}],
+        optional_params={"max_tokens": 10, "top_k": 40},
+        litellm_params={"drop_params": True},
+        headers={},
+    )
+
+    assert "top_k" not in result
+
+
+def test_top_k_raises_at_transform_without_drop_params(monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", False)
+    config = AnthropicConfig()
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
+        config.transform_request(
+            model="claude-fable-5",
+            messages=[{"role": "user", "content": "hello"}],
+            optional_params={"max_tokens": 10, "top_k": 40},
+            litellm_params={},
+            headers={},
+        )
+
+
+def test_top_k_forwarded_at_transform_on_models_that_accept_it():
+    config = AnthropicConfig()
+
+    result = config.transform_request(
+        model="claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "hello"}],
+        optional_params={"max_tokens": 10, "top_k": 40},
+        litellm_params={"drop_params": True},
+        headers={},
+    )
+
+    assert result["top_k"] == 40

tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_combined_chunk.py

@@ -0,0 +1,150 @@
+"""
+Regression tests for fake-streamed providers routed through `/v1/messages`.
+
+A fake-streaming provider (e.g. Vertex AI Gemma `:predict`) collapses its whole
+response into a single `MockResponseIterator` chunk that carries content text AND a
+`finish_reason` together. `AnthropicStreamWrapper` previously dropped all content in
+this case — `translate_streaming_openai_response_to_anthropic` sees the finish_reason
+and emits only a `message_delta`. `_CombinedChunkSplitter` splits such chunks so the
+content survives.
+"""
+
+import asyncio
+import json
+from types import SimpleNamespace
+
+from litellm.llms.anthropic.experimental_pass_through.adapters.streaming_iterator import (
+    AnthropicStreamWrapper,
+    _CombinedChunkSplitter,
+)
+from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
+from litellm.types.utils import (
+    Choices,
+    Delta,
+    Message,
+    ModelResponse,
+    ModelResponseStream,
+    StreamingChoices,
+    Usage,
+)
+
+
+def _build_fake_stream(
+    content: str, finish_reason: str = "stop"
+) -> MockResponseIterator:
+    """Mimic a Vertex Gemma `:predict` fake stream: one collapsed chunk."""
+    model_response = ModelResponse()
+    model_response.choices = [
+        Choices(
+            index=0,
+            message=Message(role="assistant", content=content),
+            finish_reason=finish_reason,
+        )
+    ]
+    model_response.usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
+    model_response.model = "gemma4"
+    return MockResponseIterator(model_response=model_response)
+
+
+def _collect_async(wrapper: AnthropicStreamWrapper) -> str:
+    async def _run() -> str:
+        out = []
+        async for raw in wrapper.async_anthropic_sse_wrapper():
+            out.append(raw.decode() if isinstance(raw, bytes) else raw)
+        return "".join(out)
+
+    return asyncio.run(_run())
+
+
+def test_fake_stream_content_reaches_anthropic_sse():
+    """Content from a collapsed fake-stream chunk must be emitted as a delta."""
+    wrapper = AnthropicStreamWrapper(
+        completion_stream=_build_fake_stream("Hello, the answer is 2."),
+        model="gemma4",
+    )
+    sse = _collect_async(wrapper)
+
+    assert "content_block_delta" in sse
+    assert "Hello, the answer is 2." in sse
+    assert "message_delta" in sse
+    assert "message_stop" in sse
+
+
+def test_fake_stream_usage_preserved():
+    """The finish chunk keeps usage so output_tokens is non-zero."""
+    wrapper = AnthropicStreamWrapper(
+        completion_stream=_build_fake_stream("Two."),
+        model="gemma4",
+    )
+    sse = _collect_async(wrapper)
+
+    message_delta = next(
+        json.loads(line[len("data: ") :])
+        for block in sse.split("\n\n")
+        for line in block.splitlines()
+        if line.startswith("data: ") and '"message_delta"' in line
+    )
+    assert message_delta["usage"]["output_tokens"] == 5
+    assert message_delta["usage"]["input_tokens"] == 10
+
+
+def test_splitter_passes_through_non_combined_chunks():
+    """A chunk with content but no finish_reason is not split."""
+    chunk = ModelResponseStream(
+        choices=[
+            StreamingChoices(
+                index=0, delta=Delta(content="partial"), finish_reason=None
+            )
+        ]
+    )
+    chunks = list(_CombinedChunkSplitter(iter([chunk])))
+    assert len(chunks) == 1
+    assert chunks[0].choices[0].delta.content == "partial"
+
+
+def test_splitter_splits_combined_chunk_into_content_then_finish():
+    """A chunk with both content and finish_reason becomes two chunks."""
+    chunk = ModelResponseStream(
+        choices=[
+            StreamingChoices(index=0, delta=Delta(content="done"), finish_reason="stop")
+        ]
+    )
+    content_chunk, finish_chunk = list(_CombinedChunkSplitter(iter([chunk])))
+
+    assert content_chunk.choices[0].delta.content == "done"
+    assert content_chunk.choices[0].finish_reason is None
+
+    assert finish_chunk.choices[0].finish_reason == "stop"
+    assert finish_chunk.choices[0].delta.content is None
+
+
+def test_is_combined_false_when_choices_empty():
+    """A metadata-only chunk with no choices is never treated as combined."""
+    assert _CombinedChunkSplitter._is_combined(SimpleNamespace(choices=[])) is False
+
+
+def test_is_combined_false_when_delta_missing():
+    """A finish chunk whose choice has no delta is not combined."""
+    chunk = SimpleNamespace(choices=[SimpleNamespace(finish_reason="stop", delta=None)])
+    assert _CombinedChunkSplitter._is_combined(chunk) is False
+
+
+def test_split_clears_reasoning_and_thinking_on_finish_chunk():
+    """When the combined delta carries reasoning/thinking, only the content
+    chunk keeps them — the finish chunk is cleared."""
+    delta = SimpleNamespace(
+        content="hi",
+        tool_calls=None,
+        reasoning_content="some reasoning",
+        thinking_blocks=[{"type": "thinking"}],
+    )
+    chunk = SimpleNamespace(
+        choices=[SimpleNamespace(finish_reason="stop", delta=delta)]
+    )
+
+    content_chunk, finish_chunk = _CombinedChunkSplitter._split(chunk)
+
+    assert content_chunk.choices[0].delta.reasoning_content == "some reasoning"
+    assert content_chunk.choices[0].delta.thinking_blocks == [{"type": "thinking"}]
+    assert finish_chunk.choices[0].delta.reasoning_content is None
+    assert finish_chunk.choices[0].delta.thinking_blocks is None

tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py

@@ -0,0 +1,79 @@
+"""
+Tests for AnthropicResponsesStreamWrapper
+(litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py)
+"""
+
+import os
+import sys
+
+sys.path.insert(
+    0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))
+)
+
+from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import (
+    AnthropicResponsesStreamWrapper,
+)
+
+
+def _process_all(events: list) -> list:
+    wrapper = AnthropicResponsesStreamWrapper(responses_stream=None, model="m")
+    for event in events:
+        wrapper._process_event(event)
+    return list(wrapper._chunk_queue)
+
+
+class TestProcessEventTextDeltaWithoutOutputItemAdded:
+    """Streams that skip response.output_item.added (e.g. LMStudio) must still
+    open a text block before any delta and never emit index -1."""
+
+    def test_process_event_synthesizes_content_block_start_before_delta(self):
+        chunks = _process_all(
+            [
+                {"type": "response.output_text.delta", "item_id": "i1", "delta": "Hel"},
+                {"type": "response.output_text.delta", "item_id": "i1", "delta": "lo"},
+            ]
+        )
+        assert [c["type"] for c in chunks] == [
+            "content_block_start",
+            "content_block_delta",
+            "content_block_delta",
+        ]
+        assert chunks[0]["content_block"] == {"type": "text", "text": ""}
+        assert [c["index"] for c in chunks] == [0, 0, 0]
+        assert chunks[1]["delta"] == {"type": "text_delta", "text": "Hel"}
+
+    def test_process_event_delta_without_item_id_never_yields_negative_index(self):
+        chunks = _process_all([{"type": "response.output_text.delta", "delta": "Hi"}])
+        assert [(c["type"], c["index"]) for c in chunks] == [
+            ("content_block_start", 0),
+            ("content_block_delta", 0),
+        ]
+
+    def test_process_event_unregistered_item_id_opens_new_text_block(self):
+        chunks = _process_all(
+            [
+                {
+                    "type": "response.output_item.added",
+                    "item": {"type": "reasoning", "id": "rs_1"},
+                },
+                {"type": "response.output_text.delta", "item_id": "m1", "delta": "Hi"},
+            ]
+        )
+        assert chunks[1]["type"] == "content_block_start"
+        assert chunks[1]["content_block"] == {"type": "text", "text": ""}
+        assert [c["index"] for c in chunks[1:]] == [1, 1]
+
+    def test_process_event_registered_item_id_does_not_synthesize_start(self):
+        chunks = _process_all(
+            [
+                {
+                    "type": "response.output_item.added",
+                    "item": {"type": "message", "id": "m1"},
+                },
+                {"type": "response.output_text.delta", "item_id": "m1", "delta": "Hi"},
+            ]
+        )
+        assert [(c["type"], c["index"]) for c in chunks] == [
+            ("content_block_start", 0),
+            ("content_block_delta", 0),
+        ]

tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py

@@ -5267,3 +5267,122 @@ def test_transform_response_does_not_leak_body_on_parse_failure():
     msg = str(exc_info.value)
     assert "secret content" not in msg
     assert "Error converting to valid response block" in msg
+
+
+def test_converse_drops_sampling_params_for_models_that_removed_them():
+    """Fable 5 / Opus 4.7 / 4.8 reject temperature != 1 and any top_p; with
+    drop_params set, converse must drop them instead of forwarding (#30064)."""
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"temperature": 0.5, "top_p": 0.9},
+        optional_params={},
+        model="us.anthropic.claude-fable-5",
+        drop_params=True,
+    )
+
+    assert "temperature" not in result
+    assert "topP" not in result
+
+
+def test_converse_sampling_params_raise_without_drop_params(monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", False)
+    config = AmazonConverseConfig()
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
+        config.map_openai_params(
+            non_default_params={"temperature": 0.5},
+            optional_params={},
+            model="global.anthropic.claude-opus-4-8-v1:0",
+            drop_params=False,
+        )
+
+
+def test_converse_sampling_params_forwarded_on_models_that_accept_them():
+    config = AmazonConverseConfig()
+
+    result = config.map_openai_params(
+        non_default_params={"temperature": 0.5, "top_p": 0.9},
+        optional_params={},
+        model="us.anthropic.claude-sonnet-4-6",
+        drop_params=True,
+    )
+
+    assert result["temperature"] == 0.5
+    assert result["topP"] == 0.9
+
+
+def test_converse_top_k_dropped_for_models_that_removed_it():
+    """``top_k`` reaches converse as a provider-specific kwarg destined for
+    ``additionalModelRequestFields``, bypassing ``map_openai_params``; the
+    transform must strip it for models that removed sampling params (#30064)."""
+    config = AmazonConverseConfig()
+
+    result = config.transform_request(
+        model="us.anthropic.claude-fable-5",
+        messages=[{"role": "user", "content": "hello"}],
+        optional_params={"top_k": 40},
+        litellm_params={"drop_params": True},
+        headers={},
+    )
+
+    assert "top_k" not in result.get("additionalModelRequestFields", {})
+
+
+def test_converse_top_k_raises_without_drop_params(monkeypatch):
+    monkeypatch.setattr(litellm, "drop_params", False)
+    config = AmazonConverseConfig()
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
+        config.transform_request(
+            model="us.anthropic.claude-fable-5",
+            messages=[{"role": "user", "content": "hello"}],
+            optional_params={"top_k": 40},
+            litellm_params={},
+            headers={},
+        )
+
+
+def test_converse_top_k_forwarded_on_models_that_accept_it():
+    config = AmazonConverseConfig()
+
+    result = config.transform_request(
+        model="us.anthropic.claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "hello"}],
+        optional_params={"top_k": 40},
+        litellm_params={"drop_params": True},
+        headers={},
+    )
+
+    assert result["additionalModelRequestFields"]["top_k"] == 40
+
+
+def test_converse_top_k_zero_raises_without_drop_params(monkeypatch):
+    """``top_k=0`` must hit the same gating as any other value; previously the
+    truthiness check let it silently disappear on models that removed sampling
+    params, diverging from the Anthropic boundary that treats ``0`` as present."""
+    monkeypatch.setattr(litellm, "drop_params", False)
+    config = AmazonConverseConfig()
+
+    with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params"):
+        config.transform_request(
+            model="us.anthropic.claude-fable-5",
+            messages=[{"role": "user", "content": "hello"}],
+            optional_params={"top_k": 0},
+            litellm_params={},
+            headers={},
+        )
+
+
+def test_converse_top_k_zero_forwarded_on_models_that_accept_it():
+    config = AmazonConverseConfig()
+
+    result = config.transform_request(
+        model="us.anthropic.claude-sonnet-4-6",
+        messages=[{"role": "user", "content": "hello"}],
+        optional_params={"top_k": 0},
+        litellm_params={"drop_params": True},
+        headers={},
+    )
+
+    assert result["additionalModelRequestFields"]["top_k"] == 0

tests/test_litellm/llms/bedrock/count_tokens/test_bedrock_count_tokens_transformation.py

@@ -1,10 +1,15 @@
+import base64
+import json
 import os
 import sys
 
 sys.path.insert(
     0, os.path.abspath("../../../../..")
 )  # Adds the parent directory to the system path
-from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
+from litellm.llms.bedrock.count_tokens.transformation import (
+    DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS,
+    BedrockCountTokensConfig,
+)
 
 
 def test_detect_input_type():
@@ -20,6 +25,71 @@ def test_detect_input_type():
     assert config._detect_input_type(request_with_text) == "invokeModel"
 
 
+def test_detect_input_type_anthropic_blocks_route_to_invoke_model():
+    """Anthropic-shape content blocks must not go through the Converse path,
+    which Bedrock rejects with a 400 (and the caller then silently falls back
+    to the local tokenizer)."""
+    config = BedrockCountTokensConfig()
+
+    request = {
+        "messages": [
+            {
+                "role": "assistant",
+                "content": [
+                    {"type": "text", "text": "Reading the file."},
+                    {
+                        "type": "tool_use",
+                        "id": "toolu_01",
+                        "name": "read_file",
+                        "input": {"path": "main.py"},
+                    },
+                ],
+            },
+        ],
+    }
+    assert config._detect_input_type(request) == "invokeModel"
+
+
+def test_detect_input_type_converse_blocks_route_to_converse():
+    """Converse-shape blocks (no "type" key) keep using the converse input."""
+    config = BedrockCountTokensConfig()
+
+    request = {"messages": [{"role": "user", "content": [{"text": "hi"}]}]}
+    assert config._detect_input_type(request) == "converse"
+
+
+def test_transform_to_invoke_model_format_base64_encodes_body():
+    """The CountTokens API expects invokeModel.body as a base64-encoded blob;
+    Anthropic Messages bodies additionally need anthropic_version/max_tokens
+    to pass Bedrock's InvokeModel schema validation."""
+    config = BedrockCountTokensConfig()
+
+    request = {
+        "model": "anthropic.claude-3-sonnet-20240229-v1:0",
+        "messages": [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}],
+    }
+
+    result = config.transform_anthropic_to_bedrock_count_tokens(request)
+
+    body = json.loads(base64.b64decode(result["input"]["invokeModel"]["body"]))
+    assert body["messages"] == request["messages"]
+    assert "model" not in body
+    assert body["anthropic_version"] == "bedrock-2023-05-31"
+    assert body["max_tokens"] == DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
+
+
+def test_transform_to_invoke_model_format_raw_body_unchanged():
+    """Non-messages bodies (e.g. Titan inputText) must not get Anthropic fields."""
+    config = BedrockCountTokensConfig()
+
+    result = config.transform_anthropic_to_bedrock_count_tokens(
+        {"model": "amazon.titan-text-express-v1", "inputText": "hello"}
+    )
+
+    body = json.loads(base64.b64decode(result["input"]["invokeModel"]["body"]))
+    assert body == {"inputText": "hello"}
+
+
 def test_transform_anthropic_to_bedrock_request():
     """Test basic request transformation"""
     config = BedrockCountTokensConfig()

tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py

@@ -1,11 +1,13 @@
 """
 Unit tests for Amazon Bedrock Mantle Responses API configuration.
 
-Mantle's gpt-5.5 / gpt-5.4 are served ONLY on the non-standard
-`/openai/v1/responses` path. These tests lock the URL construction and
-Bearer auth that make that routing work.
+Mantle serves Responses on two paths: gpt frontier models on
+`/openai/v1/responses` and other Responses-capable models (e.g. gpt-oss) on the
+standard `/v1/responses`. These tests lock the per-model path selection in the
+gate, the URL construction for both paths, and the shared Bearer auth.
 """
 
+import copy
 import os
 import sys
 
@@ -89,6 +91,42 @@ class TestBedrockMantleResponsesURL:
         url = cfg.get_complete_url(api_base=None, litellm_params={})
         assert url == "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses"
 
+    def test_standard_path_uses_region_from_env(self, monkeypatch):
+        monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2")
+        monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        url = cfg.get_complete_url(api_base=None, litellm_params={})
+        assert url == "https://bedrock-mantle.us-east-2.api.aws/v1/responses"
+        assert "/openai/v1/responses" not in url
+
+    def test_standard_path_normalizes_v1_base(self, monkeypatch):
+        monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        url = cfg.get_complete_url(
+            api_base="https://bedrock-mantle.us-east-2.api.aws/v1",
+            litellm_params={},
+        )
+        assert url == "https://bedrock-mantle.us-east-2.api.aws/v1/responses"
+        assert url.count("/responses") == 1
+        assert "/v1/v1/responses" not in url
+
+    def test_standard_path_full_endpoint_base_not_doubled(self, monkeypatch):
+        monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        url = cfg.get_complete_url(
+            api_base="https://bedrock-mantle.us-east-2.api.aws/v1/responses",
+            litellm_params={},
+        )
+        assert url == "https://bedrock-mantle.us-east-2.api.aws/v1/responses"
+        assert url.count("/responses") == 1
+
+    def test_default_construction_keeps_openai_path(self, monkeypatch):
+        monkeypatch.setenv("BEDROCK_MANTLE_REGION", "us-east-2")
+        monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False)
+        cfg = BedrockMantleResponsesAPIConfig()
+        url = cfg.get_complete_url(api_base=None, litellm_params={})
+        assert url == "https://bedrock-mantle.us-east-2.api.aws/openai/v1/responses"
+
 
 class TestBedrockMantleResponsesAuth:
     def test_config_api_key_takes_priority(self, monkeypatch):
@@ -158,6 +196,36 @@ class TestBedrockMantleResponsesAuth:
             is True
         )
 
+    def test_standard_path_still_uses_bearer_auth(self, monkeypatch):
+        monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key")
+        monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False)
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        headers = cfg.validate_environment(
+            headers={},
+            model="openai.gpt-oss-120b",
+            litellm_params=GenericLiteLLMParams(),
+        )
+        assert headers["Authorization"] == "Bearer env-key"
+
+    def test_standard_path_opts_out_of_native_features(self):
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        assert cfg.supports_native_file_search() is False
+        assert cfg.supports_native_websocket() is False
+
+
+class TestBedrockMantleResponsesRequestBody:
+    def test_standard_path_outbound_body_carries_bare_model(self):
+        cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False)
+        body = cfg.transform_responses_api_request(
+            model="openai.gpt-oss-120b",
+            input="hello",
+            response_api_optional_request_params={},
+            litellm_params=GenericLiteLLMParams(),
+            headers={},
+        )
+        assert body["model"] == "openai.gpt-oss-120b"
+        assert "input" in body
+
 
 class TestBedrockMantleResponsesRegistry:
     def test_registry_returns_config_for_gpt_5_5(self):
@@ -168,6 +236,7 @@ class TestBedrockMantleResponsesRegistry:
             model="openai.gpt-5.5",
         )
         assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is True
 
     def test_registry_returns_config_for_gpt_5_4_enum(self):
         from litellm.utils import ProviderConfigManager
@@ -177,6 +246,7 @@ class TestBedrockMantleResponsesRegistry:
             model="openai.gpt-5.4",
         )
         assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is True
 
     def test_registry_returns_none_for_gpt_oss(self):
         # Regression guard: gpt-oss must NOT get the native Responses config; it
@@ -199,9 +269,10 @@ class TestBedrockMantleResponsesRegistry:
         assert cfg is None
 
     def test_registry_returns_config_for_future_frontier_model(self):
-        # Forward-compatibility: an unseen OpenAI gpt frontier model (e.g. gpt-6) must
-        # get the native Responses config without a code change. The gate allow-lists
-        # the openai.gpt- family (minus gpt-oss), so gpt-6 matches automatically.
+        # Forward-compatibility: an unseen OpenAI gpt frontier model (e.g. gpt-6),
+        # not yet in the price map, must get the openai-path Responses config with
+        # no code or JSON change. The name-convention fallback (openai.gpt- minus
+        # gpt-oss) catches it before any price-map entry exists.
         from litellm.utils import ProviderConfigManager
 
         cfg = ProviderConfigManager.get_provider_responses_api_config(
@@ -209,6 +280,48 @@ class TestBedrockMantleResponsesRegistry:
             model="openai.gpt-6",
         )
         assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is True
+
+    def test_price_map_flag_routes_non_gpt_name_to_openai_path(
+        self, restore_model_cost
+    ):
+        # Data-driven onboarding: a frontier model whose name does NOT match the
+        # openai.gpt- convention can still be routed to /openai/v1/responses by
+        # declaring use_openai_responses_path in its price-map entry, with no code
+        # change. The string fallback alone could never catch this name.
+        from litellm.utils import ProviderConfigManager, register_model
+
+        register_model(
+            {
+                "bedrock_mantle/somelab.frontier-x": {
+                    "litellm_provider": "bedrock_mantle",
+                    "mode": "responses",
+                    "use_openai_responses_path": True,
+                }
+            }
+        )
+        cfg = ProviderConfigManager.get_provider_responses_api_config(
+            provider="bedrock_mantle",
+            model="somelab.frontier-x",
+        )
+        assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is True
+
+    def test_gpt_5_5_price_map_declares_openai_responses_path(self, local_cost_map):
+        # The gpt-5.x entries must carry the data-driven flag so frontier routing
+        # does not rely on the name-string fallback alone.
+        assert (
+            litellm.model_cost["bedrock_mantle/openai.gpt-5.5"].get(
+                "use_openai_responses_path"
+            )
+            is True
+        )
+        assert (
+            litellm.model_cost["bedrock_mantle/openai.gpt-5.4"].get(
+                "use_openai_responses_path"
+            )
+            is True
+        )
 
     @pytest.mark.parametrize(
         "model",
@@ -243,6 +356,129 @@ class TestBedrockMantleResponsesRegistry:
         )
         assert cfg is None
 
+    def test_declared_responses_non_openai_routes_to_standard_path(
+        self, restore_model_cost
+    ):
+        # New feature: a non-OpenAI model declared mode=responses (e.g. via a
+        # user's proxy model_info block) must route to the STANDARD /v1/responses
+        # path, not the frontier /openai/v1/responses path. Fails before the
+        # path-aware gate exists (old gate returned None for non-gpt models).
+        from litellm.utils import ProviderConfigManager, register_model
+
+        register_model(
+            {
+                "bedrock_mantle/somelab.future-model": {
+                    "litellm_provider": "bedrock_mantle",
+                    "mode": "responses",
+                }
+            }
+        )
+        cfg = ProviderConfigManager.get_provider_responses_api_config(
+            provider="bedrock_mantle",
+            model="somelab.future-model",
+        )
+        assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is False
+
+    def test_gpt_oss_opt_in_routes_to_standard_path(self, restore_model_cost):
+        # When a user opts gpt-oss into native Responses via model_info mode,
+        # it must take the STANDARD /v1/responses path (gpt-oss Responses is on
+        # /v1/responses, NOT the frontier /openai/v1/responses path).
+        from litellm.utils import ProviderConfigManager, register_model
+
+        register_model(
+            {
+                "bedrock_mantle/openai.gpt-oss-120b": {
+                    "litellm_provider": "bedrock_mantle",
+                    "mode": "responses",
+                }
+            }
+        )
+        cfg = ProviderConfigManager.get_provider_responses_api_config(
+            provider="bedrock_mantle",
+            model="openai.gpt-oss-120b",
+        )
+        assert isinstance(cfg, BedrockMantleResponsesAPIConfig)
+        assert cfg.use_openai_path is False
+
+    def test_unmapped_model_degrades_to_none_without_crashing(self, restore_model_cost):
+        # A non-frontier model that is not in model_cost makes get_model_info
+        # raise; the gate must swallow it and return None rather than crash.
+        from litellm.utils import ProviderConfigManager
+
+        litellm.model_cost.pop("bedrock_mantle/somelab.unmapped-model", None)
+        litellm.get_model_info.cache_clear()
+        cfg = ProviderConfigManager.get_provider_responses_api_config(
+            provider="bedrock_mantle",
+            model="somelab.unmapped-model",
+        )
+        assert cfg is None
+
+    def test_register_model_restore_undoes_existing_key_overwrite(self):
+        # Self-contained guard for the deepcopy requirement of restore_model_cost.
+        # register_model overwrites an existing key by mutating its nested dict in
+        # place, so the snapshot must be a deepcopy: a shallow dict() copy would
+        # share that nested dict and leave mode=responses after restore, making
+        # the final assertion fail. The in-place clear+update mirrors the fixture.
+        from litellm.utils import ProviderConfigManager, register_model
+
+        snapshot = copy.deepcopy(litellm.model_cost)
+        litellm.get_model_info.cache_clear()
+        try:
+            register_model(
+                {
+                    "bedrock_mantle/openai.gpt-oss-120b": {
+                        "litellm_provider": "bedrock_mantle",
+                        "mode": "responses",
+                    }
+                }
+            )
+            during = ProviderConfigManager.get_provider_responses_api_config(
+                provider="bedrock_mantle", model="openai.gpt-oss-120b"
+            )
+            assert isinstance(during, BedrockMantleResponsesAPIConfig)
+        finally:
+            litellm.model_cost.clear()
+            litellm.model_cost.update(snapshot)
+            litellm.get_model_info.cache_clear()
+        after = ProviderConfigManager.get_provider_responses_api_config(
+            provider="bedrock_mantle", model="openai.gpt-oss-120b"
+        )
+        assert after is None
+
+
+@pytest.fixture
+def restore_model_cost():
+    """Snapshot litellm.model_cost so register_model edits don't leak across tests.
+
+    register_model mutates the global litellm.model_cost, and get_model_info is
+    lru_cached, so without restore + cache_clear a registered model would bleed
+    into sibling tests in the same process.
+
+    Two subtleties make this fixture non-obvious:
+
+    1. The snapshot must be a deepcopy. register_model overwrites an existing key
+       via `litellm.model_cost.setdefault(key, {}).update(...)`, mutating the
+       nested dict in place; a shallow copy would share those nested dicts and
+       could not capture the pre-mutation values of an existing entry.
+    2. The restore must be in place (clear + update the SAME dict object), not a
+       reassignment. The conftest autouse `isolate_litellm_state` fixture
+       snapshots `litellm.model_cost` by reference and restores that reference on
+       its teardown, which runs after this one. Reassigning `litellm.model_cost`
+       to a fresh dict here is undone when conftest reinstalls its (in-place
+       mutated) reference, so the registered mode would leak and poison
+       TestBedrockMantleResponsesPricing. Mutating the original object in place
+       restores the contents conftest's reference points at.
+    """
+    original_model_cost = copy.deepcopy(litellm.model_cost)
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost.clear()
+        litellm.model_cost.update(original_model_cost)
+        litellm.get_model_info.cache_clear()
+
 
 @pytest.fixture
 def local_cost_map(monkeypatch):

tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py

@@ -0,0 +1,64 @@
+"""
+Regression test for the databricks streaming chunk parser.
+
+OpenAI-compatible servers (e.g. Vertex AI Model Garden vLLM endpoints) send a final
+usage-only chunk with an empty `choices` list when `stream_options.include_usage` is
+set. `chunk_parser` previously did `choices[0]` unconditionally, raising
+`IndexError` -> `MidStreamFallbackError` and crashing the stream.
+"""
+
+from litellm.llms.databricks.streaming_utils import ModelResponseIterator
+
+
+def test_chunk_parser_handles_empty_choices_usage_chunk():
+    """A usage-only final chunk (empty choices) must not raise IndexError."""
+    iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
+    usage_only_chunk = {
+        "id": "chatcmpl-x",
+        "object": "chat.completion.chunk",
+        "created": 1,
+        "model": "m",
+        "choices": [],
+        "usage": {"prompt_tokens": 20, "completion_tokens": 8, "total_tokens": 28},
+    }
+
+    result = iterator.chunk_parser(chunk=usage_only_chunk)
+
+    assert result["text"] == ""
+    assert result["is_finished"] is False
+    assert result["usage"] is not None
+    assert result["usage"]["prompt_tokens"] == 20
+    assert result["usage"]["completion_tokens"] == 8
+
+
+def test_chunk_parser_empty_choices_without_usage():
+    """An empty-choices chunk with no usage block returns usage=None, no error."""
+    iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
+    chunk = {
+        "id": "chatcmpl-x",
+        "object": "chat.completion.chunk",
+        "created": 1,
+        "model": "m",
+        "choices": [],
+    }
+
+    result = iterator.chunk_parser(chunk=chunk)
+
+    assert result["text"] == ""
+    assert result["usage"] is None
+
+
+def test_chunk_parser_normal_content_chunk_still_works():
+    """A regular content chunk is unaffected by the empty-choices guard."""
+    iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
+    chunk = {
+        "id": "chatcmpl-x",
+        "object": "chat.completion.chunk",
+        "created": 1,
+        "model": "m",
+        "choices": [{"index": 0, "delta": {"content": "hi"}, "finish_reason": None}],
+    }
+
+    result = iterator.chunk_parser(chunk=chunk)
+
+    assert result["text"] == "hi"

tests/test_litellm/llms/openai/completion/test_completion_handler.py

@@ -0,0 +1,93 @@
+"""
+Tests that client headers are forwarded to the provider on the OpenAI
+text completion path.
+
+Regression tests for https://github.com/BerriAI/litellm/issues/27410
+"""
+
+import os
+import sys
+
+import pytest
+import respx
+from httpx import Response
+
+sys.path.insert(0, os.path.abspath("../../../../.."))
+
+import litellm
+from litellm import atext_completion, text_completion
+
+
+@pytest.fixture(autouse=True)
+def setup_env(monkeypatch):
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-test-fake-key")
+
+
+@pytest.fixture
+def mock_completions_endpoint():
+    return respx.post("https://api.openai.com/v1/completions").mock(
+        return_value=Response(
+            200,
+            json={
+                "id": "cmpl-test123",
+                "object": "text_completion",
+                "created": 1677652288,
+                "model": "gpt-3.5-turbo-instruct",
+                "choices": [
+                    {
+                        "text": "hi",
+                        "index": 0,
+                        "logprobs": None,
+                        "finish_reason": "stop",
+                    }
+                ],
+                "usage": {
+                    "prompt_tokens": 1,
+                    "completion_tokens": 1,
+                    "total_tokens": 2,
+                },
+            },
+        )
+    )
+
+
+@respx.mock
+def test_completion_forwards_client_headers_to_provider(mock_completions_endpoint):
+    text_completion(
+        model="gpt-3.5-turbo-instruct",
+        prompt="hello",
+        max_tokens=5,
+        headers={"x-mycorp-llmcall-id": "abc-123"},
+    )
+
+    request_headers = mock_completions_endpoint.calls.last.request.headers
+    assert request_headers["x-mycorp-llmcall-id"] == "abc-123"
+
+
+@respx.mock
+def test_completion_forwards_extra_headers_to_provider(mock_completions_endpoint):
+    text_completion(
+        model="gpt-3.5-turbo-instruct",
+        prompt="hello",
+        max_tokens=5,
+        extra_headers={"x-mycorp-llmcall-id": "abc-123"},
+    )
+
+    request_headers = mock_completions_endpoint.calls.last.request.headers
+    assert request_headers["x-mycorp-llmcall-id"] == "abc-123"
+
+
+@respx.mock
+async def test_acompletion_forwards_client_headers_to_provider(
+    mock_completions_endpoint, monkeypatch
+):
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    await atext_completion(
+        model="gpt-3.5-turbo-instruct",
+        prompt="hello",
+        max_tokens=5,
+        headers={"x-mycorp-llmcall-id": "abc-123"},
+    )
+
+    request_headers = mock_completions_endpoint.calls.last.request.headers
+    assert request_headers["x-mycorp-llmcall-id"] == "abc-123"

tests/test_litellm/llms/openai_like/test_tensormesh_provider.py

@@ -2,8 +2,23 @@
 Tests for Tensormesh provider configuration and integration.
 """
 
+import pytest
+
 import litellm
 
+TENSORMESH_MODELS = [
+    "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8",
+    "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8",
+    "tensormesh/Qwen/Qwen3.6-27B-FP8",
+    "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP",
+    "tensormesh/deepseek-ai/DeepSeek-V4-Flash",
+    "tensormesh/moonshotai/Kimi-K2.6",
+    "tensormesh/MiniMaxAI/MiniMax-M2.5",
+    "tensormesh/google/gemma-4-31B-it",
+    "tensormesh/openai/gpt-oss-120b",
+    "tensormesh/openai/gpt-oss-20b",
+]
+
 
 class TestTensormeshProviderConfig:
     """Test Tensormesh provider configuration"""
@@ -82,3 +97,60 @@ class TestTensormeshProviderConfig:
 
         assert len(router.model_list) == 1
         assert router.model_list[0]["model_name"] == "tensormesh-chat"
+
+
+class TestTensormeshCostMap:
+    """The serverless models are registered in the cost map so LiteLLM can
+    price requests and unblock tool-calling params on the JSON provider path."""
+
+    @pytest.fixture(autouse=True)
+    def _use_local_model_cost_map(self, monkeypatch):
+        original_model_cost = litellm.model_cost
+        monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+        litellm.model_cost = litellm.get_model_cost_map(url="")
+        litellm.get_model_info.cache_clear()
+        try:
+            yield
+        finally:
+            litellm.model_cost = original_model_cost
+            litellm.get_model_info.cache_clear()
+
+    def test_models_registered_with_capabilities(self):
+        for model in TENSORMESH_MODELS:
+            info = litellm.get_model_info(model)
+            assert info["litellm_provider"] == "tensormesh"
+            assert info["mode"] == "chat"
+            assert litellm.supports_function_calling(model) is True, model
+            assert litellm.supports_response_schema(model) is True, model
+            assert litellm.model_cost[model]["supports_tool_choice"] is True, model
+            assert litellm.model_cost[model]["supports_prompt_caching"] is True, model
+
+    def test_reasoning_flag_matches_expected_set(self):
+        reasoning_models = {
+            "tensormesh/deepseek-ai/DeepSeek-V4-Flash",
+            "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8",
+            "tensormesh/Qwen/Qwen3.6-27B-FP8",
+            "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP",
+            "tensormesh/MiniMaxAI/MiniMax-M2.5",
+            "tensormesh/moonshotai/Kimi-K2.6",
+            "tensormesh/openai/gpt-oss-120b",
+            "tensormesh/openai/gpt-oss-20b",
+            "tensormesh/google/gemma-4-31B-it",
+        }
+        for model in TENSORMESH_MODELS:
+            assert litellm.supports_reasoning(model) is (model in reasoning_models), model
+
+    def test_cost_is_wired_and_cache_reads_are_free(self):
+        prompt_cost, completion_cost = litellm.cost_per_token(
+            model="tensormesh/openai/gpt-oss-120b",
+            prompt_tokens=1_000_000,
+            completion_tokens=1_000_000,
+        )
+        assert prompt_cost == pytest.approx(0.15)
+        assert completion_cost == pytest.approx(0.60)
+        assert (
+            litellm.model_cost["tensormesh/openai/gpt-oss-120b"][
+                "cache_read_input_token_cost"
+            ]
+            == 0
+        )

tests/test_litellm/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py

@@ -86,7 +86,7 @@ class TestContextCachingEndpoints:
         cached_content = "cached_content_123"
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = self.context_caching.check_and_create_cache(
@@ -129,7 +129,7 @@ class TestContextCachingEndpoints:
         mock_separate.return_value = ([], self.sample_messages)  # No cached messages
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = self.context_caching.check_and_create_cache(
@@ -177,7 +177,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = self.context_caching.check_and_create_cache(
@@ -254,7 +254,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = self.context_caching.check_and_create_cache(
@@ -324,7 +324,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute and Assert
         with pytest.raises(VertexAIError) as exc_info:
@@ -364,7 +364,7 @@ class TestContextCachingEndpoints:
         cached_content = "cached_content_123"
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = await self.context_caching.async_check_and_create_cache(
@@ -404,7 +404,7 @@ class TestContextCachingEndpoints:
         mock_separate.return_value = ([], self.sample_messages)
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = await self.context_caching.async_check_and_create_cache(
@@ -453,7 +453,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = await self.context_caching.async_check_and_create_cache(
@@ -535,7 +535,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute
         result = await self.context_caching.async_check_and_create_cache(
@@ -606,7 +606,7 @@ class TestContextCachingEndpoints:
 
         optional_params = self.sample_optional_params.copy()
         test_project = "test_project"
-        test_location = "test_location"
+        test_location = "us-central1"
 
         # Execute and Assert
         with pytest.raises(VertexAIError) as exc_info:
@@ -648,7 +648,7 @@ class TestContextCachingEndpoints:
             optional_params = self.sample_optional_params.copy()
             original_tools = optional_params["tools"].copy()
             test_project = "test_project"
-            test_location = "test_location"
+            test_location = "us-central1"
 
             # Mock the check_cache to return existing cache so we don't make HTTP calls
             with patch.object(
@@ -694,7 +694,7 @@ class TestContextCachingEndpoints:
             optional_params = self.sample_optional_params.copy()
             original_tools = optional_params["tools"].copy()
             test_project = "test_project"
-            test_location = "test_location"
+            test_location = "us-central1"
 
             # Execute
             result = self.context_caching.check_and_create_cache(
@@ -735,7 +735,7 @@ class TestContextCachingEndpoints:
             optional_params = self.sample_optional_params.copy()
             original_tools = optional_params["tools"].copy()
             test_project = "test_project"
-            test_location = "test_location"
+            test_location = "us-central1"
 
             # Execute
             result = await self.context_caching.async_check_and_create_cache(
@@ -778,7 +778,7 @@ class TestContextCachingEndpoints:
             optional_params = self.sample_optional_params.copy()
             original_tools = optional_params["tools"].copy()
             test_project = "test_project"
-            test_location = "test_location"
+            test_location = "us-central1"
 
             # Mock the async_check_cache to return existing cache so we don't make HTTP calls
             with patch.object(
@@ -837,7 +837,7 @@ class TestContextCachingEndpoints:
                     logging_obj=self.mock_logging,
                     custom_llm_provider=custom_llm_provider,
                     vertex_project="test_project",
-                    vertex_location="test_location",
+                    vertex_location="us-central1",
                     vertex_auth_header="vertext_test_token",
                 )
 
@@ -870,7 +870,7 @@ class TestContextCachingEndpoints:
                 logging_obj=self.mock_logging,
                 custom_llm_provider=custom_llm_provider,
                 vertex_project="test_project",
-                vertex_location="test_location",
+                vertex_location="us-central1",
                 vertex_auth_header="vertext_test_token",
             )
 
@@ -908,7 +908,7 @@ class TestContextCachingEndpoints:
                     logging_obj=self.mock_logging,
                     custom_llm_provider=custom_llm_provider,
                     vertex_project="test_project",
-                    vertex_location="test_location",
+                    vertex_location="us-central1",
                     vertex_auth_header="vertext_test_token",
                 )
 
@@ -942,7 +942,7 @@ class TestContextCachingEndpoints:
                 logging_obj=self.mock_logging,
                 custom_llm_provider=custom_llm_provider,
                 vertex_project="test_project",
-                vertex_location="test_location",
+                vertex_location="us-central1",
                 vertex_auth_header="vertext_test_token",
             )
 
@@ -1002,7 +1002,7 @@ class TestContextCachingEndpoints:
             logging_obj=self.mock_logging,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="vertext_test_token",
         )
 
@@ -1072,7 +1072,7 @@ class TestContextCachingEndpoints:
             logging_obj=self.mock_logging,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="vertext_test_token",
         )
 
@@ -1138,7 +1138,7 @@ class TestContextCachingEndpoints:
             logging_obj=self.mock_logging,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="vertext_test_token",
         )
 
@@ -1205,7 +1205,7 @@ class TestContextCachingEndpoints:
             logging_obj=self.mock_logging,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="vertext_test_token",
         )
 
@@ -1280,7 +1280,7 @@ class TestContextCachingEndpoints:
             logging_obj=self.mock_logging,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="vertext_test_token",
         )
 
@@ -1336,7 +1336,7 @@ class TestContextCachingEndpoints:
                 logging_obj=self.mock_logging,
                 custom_llm_provider=custom_llm_provider,
                 vertex_project="test_project",
-                vertex_location="test_location",
+                vertex_location="us-central1",
                 vertex_auth_header="vertext_test_token",
             )
 
@@ -1390,7 +1390,7 @@ class TestContextCachingEndpoints:
             cached_content=None,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="test_token",
         )
 
@@ -1441,7 +1441,7 @@ class TestContextCachingEndpoints:
             cached_content=None,
             custom_llm_provider=custom_llm_provider,
             vertex_project="test_project",
-            vertex_location="test_location",
+            vertex_location="us-central1",
             vertex_auth_header="test_token",
         )