Merge remote-tracking branch 'origin/litellm_internal_staging' into fix-max-parallel-requests-leak-on-stream-cancel

# Conflicts:
#	tests/test_litellm/proxy/hooks/test_parallel_request_limiter_v3.py

.circleci/config.yml

@@ -2690,6 +2690,122 @@ jobs:
           path: ui/litellm-dashboard/playwright-report
           destination: e2e-playwright-report
 
+  e2e_ui_testing_server_root_path:
+    docker:
+      - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+      - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84
+        environment:
+          POSTGRES_USER: e2euser
+          POSTGRES_PASSWORD: e2epassword
+          POSTGRES_DB: litellm_e2e
+    resource_class: large
+    working_directory: ~/project
+    environment:
+      DATABASE_URL: "postgresql://e2euser:e2epassword@localhost:5432/litellm_e2e"
+      CI: "true"
+      # The whole job exercises the proxy mounted under a prefix. SERVER_ROOT_PATH
+      # is read both by the proxy at boot (to rewrite the built UI bundle in place)
+      # and by migration.serverRootPath.config.ts, which refuses to run without it.
+      SERVER_ROOT_PATH: "/litellm"
+    steps:
+      - checkout
+      - setup_google_dns
+      - install_uv
+      - restore_cache:
+          keys:
+            - v1-uv-cache-{{ checksum "uv.lock" }}
+      - run:
+          name: Install Python dependencies
+          command: |
+            uv sync --frozen --all-groups --all-extras --python 3.12
+            uv run --no-sync python -m prisma generate --schema litellm/proxy/schema.prisma
+      - save_cache:
+          key: v1-uv-cache-{{ checksum "uv.lock" }}
+          paths:
+            - ~/.cache/uv
+      - restore_cache:
+          keys:
+            - ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+      - run:
+          name: Install Node dependencies and Playwright
+          command: |
+            cd ui/litellm-dashboard
+            npm ci
+            npx playwright install chromium
+      - save_cache:
+          key: ui-e2e-node-deps-v2-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
+          paths:
+            - ui/litellm-dashboard/node_modules
+            - ~/.cache/ms-playwright
+      - run:
+          name: Build UI from source
+          command: |
+            cd ui/litellm-dashboard
+            npm run build
+            rm -rf ../../litellm/proxy/_experimental/out
+            mv out ../../litellm/proxy/_experimental/out
+            find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
+              d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
+            done
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "30"
+      - run:
+          name: Push Prisma schema
+          command: uv run --no-sync python -m prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss
+      - run:
+          name: Seed database
+          command: |
+            PGPASSWORD=e2epassword psql -h localhost -p 5432 -U e2euser -d litellm_e2e \
+              -f ui/litellm-dashboard/e2e_tests/fixtures/seed.sql
+      - run:
+          name: Start mock LLM server
+          command: uv run --no-sync python ui/litellm-dashboard/e2e_tests/fixtures/mock_llm_server/server.py
+          background: true
+      - run:
+          name: Start LiteLLM proxy under a server root path
+          environment:
+            LITELLM_MASTER_KEY: "sk-1234"
+            MOCK_LLM_URL: "http://127.0.0.1:8090/v1"
+            DISABLE_SCHEMA_UPDATE: "true"
+          # Output flows to this step's own log, so a boot crash is visible here
+          # rather than swallowed by a downstream readiness probe.
+          command: |
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              uv run --no-sync python -m litellm.proxy.proxy_cli \
+                --config ui/litellm-dashboard/e2e_tests/fixtures/config.yml \
+                --port 4000
+          background: true
+      - run:
+          name: Wait for prefixed proxy to be ready
+          command: |
+            for i in $(seq 1 60); do
+              HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 -H "Authorization: Bearer sk-1234" http://127.0.0.1:4000/litellm/health 2>/dev/null || true)
+              if [ "$HTTP_CODE" = "200" ]; then
+                echo "Prefixed proxy is ready"
+                exit 0
+              fi
+              sleep 2
+            done
+            echo "Prefixed proxy failed to start; see the 'Start LiteLLM proxy under a server root path' step for the boot log"
+            exit 1
+      - run:
+          name: Run migration smoke under SERVER_ROOT_PATH
+          command: |
+            cd ui/litellm-dashboard
+            LITELLM_LICENSE="$LITELLM_LICENSE" \
+              npx playwright test --config e2e_tests/migration.serverRootPath.config.ts
+          no_output_timeout: 10m
+      - store_artifacts:
+          path: ui/litellm-dashboard/test-results
+          destination: e2e-server-root-path-test-results
+      - store_artifacts:
+          path: ui/litellm-dashboard/playwright-report
+          destination: e2e-server-root-path-playwright-report
+
   build_docker_database_image:
     machine:
       image: ubuntu-2204:2024.04.1
@@ -2795,6 +2911,8 @@ workflows:
           filters: *main_branches
       - e2e_ui_testing:
           filters: *main_branches
+      - e2e_ui_testing_server_root_path:
+          filters: *main_branches
       - build_and_test:
           requires:
             - build_docker_database_image

.github/workflows/create-release.yml

@@ -52,6 +52,22 @@ jobs:
             // are stable maintenance releases, not pre-releases.
             const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
 
+            // A stable release should only claim the repo "latest" badge when its
+            // version is >= the current latest. Otherwise a backport (e.g. 1.84.6)
+            // would steal "latest" from a newer line (e.g. 1.88.1).
+            const versionKey = (rawTag) => {
+              const m = String(rawTag).match(/^v?(\d+)\.(\d+)\.(\d+)/);
+              if (!m) return null;
+              const maintenance = String(rawTag).match(/(?:\.post|\.patch\.)(\d+)/i);
+              return [Number(m[1]), Number(m[2]), Number(m[3]), maintenance ? Number(maintenance[1]) : 0];
+            };
+            const isAtLeast = (a, b) => {
+              for (let i = 0; i < a.length; i++) {
+                if (a[i] !== b[i]) return a[i] > b[i];
+              }
+              return true;
+            };
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -90,6 +106,22 @@ jobs:
             ].join('\n');
 
             try {
+              let makeLatest = "false";
+              const newVersion = versionKey(tag);
+              if (!isPrerelease && newVersion) {
+                let latestVersion = null;
+                try {
+                  const latest = await github.rest.repos.getLatestRelease({
+                    owner: context.repo.owner,
+                    repo: context.repo.repo,
+                  });
+                  latestVersion = versionKey(latest.data.tag_name);
+                } catch (error) {
+                  if (error.status !== 404) throw error;
+                }
+                makeLatest = (!latestVersion || isAtLeast(newVersion, latestVersion)) ? "true" : "false";
+              }
+
               const response = await github.rest.repos.createRelease({
                 draft: true,
                 generate_release_notes: true,
@@ -108,6 +140,7 @@ jobs:
                 release_id: response.data.id,
                 body: updatedBody,
                 draft: false,
+                make_latest: makeLatest,
               });
 
             } catch (error) {

CLAUDE.md

@@ -52,6 +52,19 @@ Do not put names of customers or customer company names in code, PRs, and issues
 
 CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bash`, `wget ... | sh`); download the artifact to a file, verify its SHA-256 checksum, then install. Pin every external tool to a specific version with a full URL (not `latest` or `stable`). Verify checksums for all downloaded binaries, using the provider's official `.sha256` / `.sha256sum` sidecar when available. These rules apply to every download in CI
 
+Follow these coding conventions for new/updated code (a three-line fix in a legacy file shouldn't trigger huge drive-by refactors):
+
+- Composition over inheritance
+- Never-nester: early returns over deep nesting
+- Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
+- No mutation; instead of mutable lists and dicts, prefer tuples, NamedTuples, frozen dataclasses, etc.
+- Use dependency injection
+- Fully typed; no `Any` or coarse types like dict[str, Any]. Every function parameter must be strongly typed
+- Use tagged unions + match
+- No monster files or god objects
+
+Follow conventional commits for commit names and PR titles
+
 ## Think Before Coding
 
 **Don't assume. Don't hide confusion. Surface tradeoffs**

litellm/anthropic_beta_headers_config.json

@@ -129,7 +129,7 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": null,
-    "compact-2026-01-12": null,
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",

litellm/caching/caching.py

@@ -691,6 +691,7 @@ class Cache:
         self,
         embedding_response: Any,
         model: Optional[str],
+        prompt_tokens: Optional[int] = None,
         prompt_tokens_details: Optional[dict] = None,
     ) -> CachedEmbedding:
         """
@@ -703,6 +704,7 @@ class Cache:
                     "index": embedding_response.get("index"),
                     "object": embedding_response.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
             elif hasattr(embedding_response, "model_dump"):
@@ -712,6 +714,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
             else:
@@ -721,6 +724,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens": prompt_tokens,
                     "prompt_tokens_details": prompt_tokens_details,
                 }
         except KeyError as e:
@@ -769,6 +773,29 @@ class Cache:
                 per_item[key] = value
         return per_item if per_item else None
 
+    def _get_per_item_prompt_tokens(
+        self,
+        result: EmbeddingResponse,
+        idx_in_result_data: int,
+    ) -> Optional[int]:
+        """
+        Extract the per-item prompt_tokens from a response for caching.
+
+        Single-item responses store the full usage.prompt_tokens. Multi-item
+        responses distribute it evenly (with remainder) so that summing all
+        per-item values on retrieval reconstructs the original total.
+        """
+        if result.usage is None or result.usage.prompt_tokens is None:
+            return None
+
+        total = result.usage.prompt_tokens
+        num_items = len(result.data)
+        if num_items <= 1:
+            return total
+
+        quotient, remainder = divmod(total, num_items)
+        return quotient + (1 if idx_in_result_data < remainder else 0)
+
     def add_embedding_response_to_cache(
         self,
         result: EmbeddingResponse,
@@ -780,7 +807,11 @@ class Cache:
         kwargs["cache_key"] = preset_cache_key
         embedding_response = result.data[idx_in_result_data]
 
-        # Extract per-item prompt_tokens_details from response usage
+        # Extract per-item prompt_tokens + details from response usage
+        prompt_tokens = self._get_per_item_prompt_tokens(
+            result=result,
+            idx_in_result_data=idx_in_result_data,
+        )
         prompt_tokens_details = self._get_per_item_prompt_tokens_details(
             result=result,
             idx_in_result_data=idx_in_result_data,
@@ -791,6 +822,7 @@ class Cache:
         embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
             embedding_response,
             model_name,
+            prompt_tokens=prompt_tokens,
             prompt_tokens_details=prompt_tokens_details,
         )
 

litellm/caching/caching_handler.py

@@ -394,7 +394,7 @@ class LLMCachingHandler:
                 return cr["model"]
         return None
 
-    def _process_async_embedding_cached_response(
+    def _process_async_embedding_cached_response(  # noqa: PLR0915
         self,
         final_embedding_cached_response: Optional[EmbeddingResponse],
         cached_result: List[Optional[CachedEmbedding]],
@@ -456,7 +456,10 @@ class LLMCachingHandler:
                             index=idx,
                             object="embedding",
                         )
-                    if isinstance(kwargs_input_as_list[idx], str):
+                    cached_prompt_tokens = cr.get("prompt_tokens")
+                    if cached_prompt_tokens is not None:
+                        prompt_tokens += cached_prompt_tokens
+                    elif isinstance(kwargs_input_as_list[idx], str):
                         from litellm.utils import token_counter
 
                         prompt_tokens += token_counter(

litellm/constants.py

@@ -1158,6 +1158,7 @@ BEDROCK_CONVERSE_MODELS = [
     "openai.gpt-oss-120b-1:0",
     "anthropic.claude-haiku-4-5-20251001-v1:0",
     "anthropic.claude-sonnet-4-5-20250929-v1:0",
+    "anthropic.claude-fable-5",
     "anthropic.claude-opus-4-8",
     "anthropic.claude-opus-4-7",
     "anthropic.claude-opus-4-6-v1:0",

litellm/integrations/compression_interception/handler.py

@@ -72,8 +72,13 @@ class CompressionInterceptionLogger(CustomLogger):
         compression_params: CompressionInterceptionConfig = {}
         if "compression_interception_params" in litellm_settings:
             compression_params = litellm_settings["compression_interception_params"]
-        elif "compression_interception" in callback_specific_params:
-            compression_params = callback_specific_params["compression_interception"]
+        elif "compression_interception" in callback_specific_params and isinstance(
+            callback_specific_params["compression_interception"], dict
+        ):
+            compression_params = cast(
+                CompressionInterceptionConfig,
+                callback_specific_params["compression_interception"],
+            )
         return CompressionInterceptionLogger.from_config_yaml(compression_params)
 
     async def async_pre_call_deployment_hook(

litellm/integrations/websearch_interception/handler.py

@@ -1339,8 +1339,13 @@ class WebSearchInterceptionLogger(CustomLogger):
         websearch_params: WebSearchInterceptionConfig = {}
         if "websearch_interception_params" in litellm_settings:
             websearch_params = litellm_settings["websearch_interception_params"]
-        elif "websearch_interception" in callback_specific_params:
-            websearch_params = callback_specific_params["websearch_interception"]
+        elif "websearch_interception" in callback_specific_params and isinstance(
+            callback_specific_params["websearch_interception"], dict
+        ):
+            websearch_params = cast(
+                WebSearchInterceptionConfig,
+                callback_specific_params["websearch_interception"],
+            )
 
         # Use classmethod to initialize from config
         return WebSearchInterceptionLogger.from_config_yaml(websearch_params)

litellm/litellm_core_utils/cli_token_utils.py

@@ -37,7 +37,7 @@ def get_litellm_gateway_api_key(
     """
     Get the stored CLI API key for use with LiteLLM SDK.
 
-    This function reads the token file created by `litellm-proxy login`
+    This function reads the token file created by `lite login`
     and returns the API key for use in Python scripts.
 
     Args:

litellm/litellm_core_utils/get_litellm_params.py

@@ -32,8 +32,10 @@ _OPTIONAL_KWARGS_KEYS = frozenset(
         "aws_sts_endpoint",
         "aws_external_id",
         "aws_bedrock_runtime_endpoint",
+        "aws_bedrock_project_id",
         "tpm",
         "rpm",
+        "use_xai_oauth",
     }
 )
 

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple
 
 import litellm
 from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
+from litellm.litellm_core_utils.llm_cost_calc.utils import _get_web_search_requests
 from litellm.types.llms.openai import (
     FileSearchTool,
     ResponsesAPIResponse,
@@ -339,8 +340,7 @@ class StandardBuiltInToolCostTracking:
                 # and _handle_web_search_cost() is never called.
                 if (
                     hasattr(usage, "server_tool_use")
-                    and usage.server_tool_use is not None
-                    and usage.server_tool_use.web_search_requests is not None
+                    and _get_web_search_requests(usage.server_tool_use) is not None
                 ):
                     return True
             return False
@@ -352,8 +352,7 @@ class StandardBuiltInToolCostTracking:
         elif usage is not None:
             if (
                 hasattr(usage, "server_tool_use")
-                and usage.server_tool_use is not None
-                and usage.server_tool_use.web_search_requests is not None
+                and _get_web_search_requests(usage.server_tool_use) is not None
             ):
                 return True
             elif (

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1,7 +1,7 @@
 # What is this?
 ## Helper utilities for cost_per_token()
 
-from typing import Literal, Optional, Tuple, TypedDict, cast
+from typing import Any, Literal, Optional, Tuple, TypedDict, cast
 
 import litellm
 from litellm._logging import verbose_logger
@@ -42,6 +42,26 @@ def _get_token_detail_value(details: object, key: str) -> Optional[int]:
     return value if isinstance(value, int) else None
 
 
+def _get_web_search_requests(server_tool_use: Any) -> Optional[int]:
+    """
+    Tolerantly read ``web_search_requests`` from a ``server_tool_use`` value
+    that may be ``None``, a ``dict``, a ``ServerToolUse`` pydantic instance,
+    or any other object supporting attribute access.
+
+    Returns ``None`` when the value cannot be resolved — callers can
+    distinguish "absent" from "zero" using ``is None``.
+
+    See https://github.com/BerriAI/litellm/issues/26153 — ``stream_chunk_builder``
+    historically left this as a plain ``dict``, which broke direct attribute
+    access in cost calculation.
+    """
+    if server_tool_use is None:
+        return None
+    if isinstance(server_tool_use, dict):
+        return server_tool_use.get("web_search_requests")
+    return getattr(server_tool_use, "web_search_requests", None)
+
+
 def _is_above_128k(tokens: float) -> bool:
     if tokens > 128000:
         return True

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -633,11 +633,6 @@ def convert_to_model_response_object(  # noqa: PLR0915
                         thinking_blocks = choice["message"]["thinking_blocks"]
                         provider_specific_fields["thinking_blocks"] = thinking_blocks
 
-                    if reasoning_content:
-                        provider_specific_fields["reasoning_content"] = (
-                            reasoning_content
-                        )
-
                     message = Message(
                         content=content,
                         role=choice["message"]["role"] or "assistant",

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -4290,6 +4290,49 @@ def _deduplicate_bedrock_tool_content(
     return _deduplicate_bedrock_content_blocks(tool_content, "toolResult")
 
 
+def _rename_duplicate_bedrock_document_names(
+    contents: List[BedrockMessageBlock],
+) -> List[BedrockMessageBlock]:
+    """
+    Rename duplicate document names across all messages in a Bedrock request.
+
+    Document names are derived from a content hash, so the same file appearing
+    in multiple conversation turns produces identical names and Bedrock rejects
+    the request with "Messages can not contain duplicate document names".  The
+    first occurrence keeps its original name so prompt-cache prefixes stay
+    stable; later occurrences get a deterministic positional suffix
+    (``_2``, ``_3``, ...), bumped further if the suffixed name already
+    belongs to another document (e.g. an organic name ending in ``_2``).
+    """
+    used_names: Set[str] = set()
+    for message in contents:
+        for block in message.get("content") or []:
+            document = block.get("document")
+            if isinstance(document, dict) and document.get("name"):
+                used_names.add(document["name"])
+
+    name_counts: Dict[str, int] = {}
+    for message in contents:
+        for block in message.get("content") or []:
+            document = block.get("document")
+            if not isinstance(document, dict):
+                continue
+            name = document.get("name")
+            if not name:
+                continue
+            count = name_counts.get(name, 0) + 1
+            name_counts[name] = count
+            if count > 1:
+                suffix = count
+                new_name = f"{name}_{suffix}"
+                while new_name in used_names:
+                    suffix += 1
+                    new_name = f"{name}_{suffix}"
+                used_names.add(new_name)
+                document["name"] = new_name
+    return contents
+
+
 def _sort_bedrock_assistant_content_blocks(
     blocks: List[BedrockContentBlock],
 ) -> List[BedrockContentBlock]:
@@ -4938,7 +4981,7 @@ class BedrockConverseMessagesProcessor:
                     llm_provider=llm_provider,
                 )
 
-        return contents
+        return _rename_duplicate_bedrock_document_names(contents)
 
     @staticmethod
     def translate_thinking_blocks_to_reasoning_content_blocks(
@@ -5360,7 +5403,7 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                 llm_provider=llm_provider,
             )
 
-    return contents
+    return _rename_duplicate_bedrock_document_names(contents)
 
 
 def make_valid_bedrock_tool_name(input_tool_name: str) -> str:

litellm/litellm_core_utils/realtime_streaming.py

@@ -144,7 +144,7 @@ class RealTimeStreaming:
             return True
         return False
 
-    def store_message(self, message: Union[str, bytes, OpenAIRealtimeEvents]):
+    def store_message(self, message: Union[str, bytes, dict, OpenAIRealtimeEvents]):
         """Store message in list"""
         if isinstance(message, bytes):
             message = message.decode("utf-8")
@@ -154,22 +154,20 @@ class RealTimeStreaming:
         else:
             message_obj = cast(Dict[str, Any], json.loads(cast(str, message)))
         self._collect_tool_calls_from_response_done(cast(dict, message_obj))
+        if not self._should_store_message(message_obj):
+            return
         try:
             event_type = message_obj.get("type", "")
             if event_type in self._SESSION_EVENT_TYPES:
-                typed_obj = OpenAIRealtimeStreamSessionEvents(**message_obj)  # type: ignore
+                typed_obj: OpenAIRealtimeEvents = OpenAIRealtimeStreamSessionEvents(**message_obj)  # type: ignore
             else:
-                # Use the base object as a safe catch-all for all other event types
-                # (both beta and GA), so unknown/new event names never raise here.
+                # Catch-all base object so unknown/new event names never raise.
                 typed_obj = OpenAIRealtimeStreamResponseBaseObject(**message_obj)  # type: ignore
         except Exception as e:
             verbose_logger.debug(f"Error parsing message for logging: {e}")
-            # Don't re-raise — a parse failure must not drop or delay the message
-            if self._should_store_message(message_obj):
-                self.messages.append(message_obj)  # type: ignore[arg-type]
+            self.messages.append(message_obj)  # type: ignore[arg-type]
             return
-        if self._should_store_message(typed_obj):
-            self.messages.append(typed_obj)
+        self.messages.append(typed_obj)
 
     def _collect_user_input_from_client_event(self, message: Union[str, dict]) -> None:
         """Extract user text content from client WebSocket events for spend logging."""
@@ -358,8 +356,7 @@ class RealTimeStreaming:
                     for msg in self._pending_messages_until_setup
                 )
                 verbose_logger.debug(
-                    "Failed to flush buffered client message after setup: %s "
-                    "(%d buffered message(s) retained)",
+                    "Failed to flush buffered client message after setup: %s (%d buffered message(s) retained)",
                     e,
                     len(unsent),
                 )
@@ -376,8 +373,7 @@ class RealTimeStreaming:
                 return True
             except Exception as e:
                 verbose_logger.warning(
-                    "Failed to translate %s to beta protocol, forwarding "
-                    "untranslated event to client: %s",
+                    "Failed to translate %s to beta protocol, forwarding untranslated event to client: %s",
                     event.get("type"),
                     e,
                 )
@@ -705,48 +701,48 @@ class RealTimeStreaming:
             self.store_message(event_str)
             await self._send_event_to_client(event, event_str)
 
-    async def _handle_raw_backend_message(self, raw_response) -> bool:
+    @staticmethod
+    def _parse_backend_event(raw_response: str) -> Optional[dict]:
+        """Parse a backend frame once. Returns None for non-JSON or non-object frames."""
+        try:
+            event = json.loads(raw_response)
+        except (json.JSONDecodeError, TypeError):
+            return None
+        return event if isinstance(event, dict) else None
+
+    async def _handle_raw_backend_message(
+        self, event_obj: dict, raw_response: str
+    ) -> bool:
         """Process a backend message without provider_config (raw path).
 
         Returns True if the caller should skip the default store+forward (i.e. continue the loop).
         """
-        try:
-            event_obj = json.loads(raw_response)
+        event_type = event_obj.get("type")
 
-            # For audio/VAD guardrail path: once the session is ready, tell the backend
-            # not to auto-respond after VAD detects end-of-speech.  We send the
-            # session.created to the client FIRST so the client is always in sync, then
-            # inject the session.update so a potential error from the backend doesn't
-            # arrive before the client sees session.created.
-            if (
-                event_obj.get("type") == "session.created"
-                and self._has_audio_transcription_guardrails()
-            ):
-                self.store_message(raw_response)
-                await self.websocket.send_text(raw_response)
-                await self._send_to_backend(self._make_disable_auto_response_message())
-                return True
+        # Send session.created to the client FIRST so it stays in sync, then inject
+        # the disable-auto-response session.update; otherwise a backend error could
+        # reach the client before it sees session.created.
+        if (
+            event_type == "session.created"
+            and self._has_audio_transcription_guardrails()
+        ):
+            self.store_message(event_obj)
+            await self.websocket.send_text(raw_response)
+            await self._send_to_backend(self._make_disable_auto_response_message())
+            return True
 
-            if (
-                event_obj.get("type")
-                == "conversation.item.input_audio_transcription.completed"
-            ):
-                transcript = event_obj.get("transcript", "")
-                self._collect_user_input_from_backend_event(event_obj)
-                ## LOGGING — must happen before continue below
-                self.store_message(raw_response)
-                # Forward transcript to client so user sees what they said
-                await self.websocket.send_text(raw_response)
-                blocked = await self.run_realtime_guardrails(
-                    transcript,
-                    item_id=event_obj.get("item_id"),
-                )
-                if not blocked:
-                    # Clean — trigger LLM response
-                    await self._send_to_backend(json.dumps({"type": "response.create"}))
-                return True
-        except (json.JSONDecodeError, AttributeError):
-            pass
+        if event_type == "conversation.item.input_audio_transcription.completed":
+            transcript = event_obj.get("transcript", "")
+            self._collect_user_input_from_backend_event(event_obj)
+            self.store_message(event_obj)
+            await self.websocket.send_text(raw_response)
+            blocked = await self.run_realtime_guardrails(
+                transcript,
+                item_id=event_obj.get("item_id"),
+            )
+            if not blocked:
+                await self._send_to_backend(json.dumps({"type": "response.create"}))
+            return True
         return False
 
     async def backend_to_client_send_messages(self):
@@ -779,25 +775,25 @@ class RealTimeStreaming:
                         )
                         continue
                 else:
-                    handled = await self._handle_raw_backend_message(raw_response)
-                    if handled:
-                        continue
-                    ## LOGGING
-                    self.store_message(raw_response)
-
-                    # If the client opted into beta protocol, translate GA event
-                    # names/shapes back to the beta equivalents before forwarding.
-                    if self._client_wants_beta:
-                        try:
-                            event_dict = json.loads(raw_response)
-                            translated = self._translate_event_to_beta(event_dict)
-                            if translated is None:
-                                continue  # drop GA-only events (e.g. conversation.item.done)
-                            await self.websocket.send_text(json.dumps(translated))
-                        except Exception:
-                            await self.websocket.send_text(raw_response)
-                    else:
+                    event = self._parse_backend_event(raw_response)
+                    if event is None:
                         await self.websocket.send_text(raw_response)
+                        continue
+
+                    if await self._handle_raw_backend_message(event, raw_response):
+                        continue
+                    self.store_message(event)
+
+                    if not self._client_wants_beta:
+                        await self.websocket.send_text(raw_response)
+                        continue
+
+                    translated = self._translate_event_to_beta(event)
+                    if translated is None:
+                        continue
+                    await self.websocket.send_text(
+                        raw_response if translated is event else json.dumps(translated)
+                    )
 
         except websockets.exceptions.ConnectionClosed as e:  # type: ignore
             verbose_logger.exception(
@@ -927,41 +923,43 @@ class RealTimeStreaming:
     def _translate_event_to_beta(event: dict) -> Optional[dict]:
         """Translate a single GA event dict to its beta equivalent.
 
-        Returns None if the event should be dropped entirely (e.g. the GA-only
-        conversation.item.done has no beta counterpart).
-        Returns the (possibly mutated copy of the) event otherwise.
+        Returns None when the event must be dropped (the GA-only
+        conversation.item.done has no beta counterpart). Returns the original
+        event object unchanged when no translation applies, so the caller can
+        forward the raw frame without re-serializing; otherwise returns a
+        translated copy.
         """
         event_type = event.get("type", "")
 
-        # conversation.item.done has no beta equivalent — the client already
-        # received conversation.item.created (translated from .added).
         if event_type == "conversation.item.done":
             return None
 
-        # Shallow-copy so we don't mutate the stored message
+        renamed_type = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES.get(event_type)
+        has_item = isinstance(event.get("item"), dict)
+        response = event.get("response")
+        has_response_output = isinstance(response, dict) and isinstance(
+            response.get("output"), list
+        )
+        if renamed_type is None and not has_item and not has_response_output:
+            return event
+
         translated = dict(event)
-
-        # Rename the type field
-        if event_type in RealTimeStreaming._GA_TO_BETA_EVENT_TYPES:
-            translated["type"] = RealTimeStreaming._GA_TO_BETA_EVENT_TYPES[event_type]
-
-        # Fix content block types inside items (response.done output list,
-        # conversation.item.created item content, etc.)
-        if "item" in translated and isinstance(translated["item"], dict):
+        if renamed_type is not None:
+            translated["type"] = renamed_type
+        if has_item:
             translated["item"] = RealTimeStreaming._translate_item_content_types(
                 dict(translated["item"])
             )
-        if "response" in translated and isinstance(translated["response"], dict):
+        if has_response_output:
             resp = dict(translated["response"])
-            if "output" in resp and isinstance(resp["output"], list):
-                resp["output"] = [
-                    (
-                        RealTimeStreaming._translate_item_content_types(dict(o))
-                        if isinstance(o, dict)
-                        else o
-                    )
-                    for o in resp["output"]
-                ]
+            resp["output"] = [
+                (
+                    RealTimeStreaming._translate_item_content_types(dict(o))
+                    if isinstance(o, dict)
+                    else o
+                )
+                for o in resp["output"]
+            ]
             translated["response"] = resp
 
         return translated

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -637,7 +637,18 @@ class ChunkProcessor:
                     hasattr(usage_chunk, "server_tool_use")
                     and usage_chunk.server_tool_use is not None
                 ):
-                    server_tool_use = usage_chunk.server_tool_use
+                    # Coerce dict to ServerToolUse so downstream cost-calc code
+                    # (which accesses .web_search_requests as an attribute)
+                    # doesn't raise AttributeError. Some providers / streaming
+                    # paths leave server_tool_use as a plain dict on the chunk.
+                    if isinstance(usage_chunk.server_tool_use, dict):
+                        server_tool_use = ServerToolUse(**usage_chunk.server_tool_use)
+                    elif isinstance(usage_chunk.server_tool_use, ServerToolUse):
+                        server_tool_use = usage_chunk.server_tool_use
+                    else:
+                        server_tool_use = ServerToolUse.model_validate(
+                            usage_chunk.server_tool_use
+                        )
                 if (
                     usage_chunk_dict["prompt_tokens_details"] is not None
                     and getattr(

litellm/llms/anthropic/chat/transformation.py

@@ -1455,10 +1455,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 _value = self._map_stop_sequences(value)
                 if _value is not None:
                     optional_params["stop_sequences"] = _value
-            elif param == "temperature":
-                optional_params["temperature"] = value
-            elif param == "top_p":
-                optional_params["top_p"] = value
+            elif param == "temperature" or param == "top_p":
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=optional_params,
+                    model=model,
+                    param=param,
+                    value=value,
+                    drop_params=drop_params,
+                    output_key=param,
+                )
             elif param == "response_format" and isinstance(value, dict):
                 if any(
                     substring in model
@@ -1975,6 +1980,20 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         optional_params.pop("is_vertex_request", None)
         optional_params.pop("client_metadata", None)
 
+        # ``top_k`` is a provider-specific kwarg that bypasses
+        # ``map_openai_params``; gate it here, the single boundary shared by
+        # the direct Anthropic, Bedrock invoke, Vertex, and Azure paths.
+        top_k = optional_params.pop("top_k", None)
+        if top_k is not None:
+            AnthropicConfig._apply_sampling_param(
+                optional_params=optional_params,
+                model=model,
+                param="top_k",
+                value=top_k,
+                drop_params=litellm_params.get("drop_params") is True,
+                output_key="top_k",
+            )
+
         data = {
             "model": model,
             "messages": anthropic_messages,

litellm/llms/anthropic/common_utils.py

@@ -272,23 +272,68 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         )
 
     @staticmethod
-    def _supports_model_capability(model: str, key: str) -> bool:
-        """Check a boolean capability ``key`` in the model map.
+    def _supports_sampling_params(model: str) -> bool:
+        """Claude 4.7+ (Opus 4.7/4.8, Fable 5) removed sampling params: the API
+        rejects ``top_p``, ``top_k``, and any ``temperature`` other than 1 with
+        a 400 ("`temperature` is deprecated for this model").
 
-        Strips bedrock/vertex prefixes so a provider-routed Claude still
-        resolves to the Anthropic model-map entry.
-        """
-        from litellm.utils import _supports_factory
+        Driven by the ``supports_sampling_params`` flag in the model map; the
+        name check remains only as a fallback for provider-routed ids whose
+        map entries predate the flag."""
+        flag = AnthropicModelInfo._get_model_capability(
+            model, "supports_sampling_params"
+        )
+        if flag is not None:
+            return flag
+        model_lower = model.lower()
+        return not any(
+            v in model_lower
+            for v in (
+                "fable",
+                "opus-4-7",
+                "opus_4_7",
+                "opus-4.7",
+                "opus_4.7",
+                "opus-4-8",
+                "opus_4_8",
+                "opus-4.8",
+                "opus_4.8",
+            )
+        )
 
-        try:
-            if _supports_factory(
-                model=model,
-                custom_llm_provider="anthropic",
-                key=key,
-            ):
-                return True
-        except Exception:
-            pass
+    @staticmethod
+    def _apply_sampling_param(
+        optional_params: dict,
+        model: str,
+        param: str,
+        value: Any,
+        drop_params: bool,
+        output_key: str,
+    ) -> None:
+        """Forward ``temperature``/``top_p``/``top_k`` to
+        ``optional_params[output_key]`` unless the model removed sampling
+        params, in which case drop the param (with drop_params) or raise a
+        clean client-side 400."""
+        if AnthropicModelInfo._supports_sampling_params(model) or (
+            param == "temperature" and value == 1
+        ):
+            optional_params[output_key] = value
+        elif not (litellm.drop_params or drop_params):
+            supported_hint = (
+                "Only temperature=1 is supported. " if param == "temperature" else ""
+            )
+            raise litellm.utils.UnsupportedParamsError(
+                message=(
+                    f"{model} does not support {param}={value}. {supported_hint}"
+                    "To drop unsupported params, set `litellm.drop_params = True`."
+                ),
+                status_code=400,
+            )
+
+    @staticmethod
+    def _model_map_lookup_candidates(model: str) -> List[str]:
+        """Model-map keys to try for ``model``, stripping bedrock/vertex
+        prefixes so a provider-routed Claude still resolves to its entry."""
         candidates = [model]
         for prefix in (
             "bedrock/converse/",
@@ -307,15 +352,40 @@ class AnthropicModelInfo(BaseLLMModelInfo):
                 candidates.append(f"bedrock/{base}")
         except Exception:
             pass
+        return candidates
+
+    @staticmethod
+    def _get_model_capability(model: str, key: str) -> Optional[bool]:
+        """Read boolean capability ``key`` from the model map, or None when
+        no entry declares it."""
         try:
-            for cand in candidates:
-                if cand in litellm.model_cost and (
-                    litellm.model_cost[cand].get(key) is True
-                ):
-                    return True
+            for cand in AnthropicModelInfo._model_map_lookup_candidates(model):
+                value = litellm.model_cost.get(cand, {}).get(key)
+                if isinstance(value, bool):
+                    return value
         except Exception:
             pass
-        return False
+        return None
+
+    @staticmethod
+    def _supports_model_capability(model: str, key: str) -> bool:
+        """Check a boolean capability ``key`` in the model map.
+
+        Strips bedrock/vertex prefixes so a provider-routed Claude still
+        resolves to the Anthropic model-map entry.
+        """
+        from litellm.utils import _supports_factory
+
+        try:
+            if _supports_factory(
+                model=model,
+                custom_llm_provider="anthropic",
+                key=key,
+            ):
+                return True
+        except Exception:
+            pass
+        return AnthropicModelInfo._get_model_capability(model, key) is True
 
     @staticmethod
     def _is_adaptive_thinking_model(model: str) -> bool:

litellm/llms/anthropic/cost_calculation.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Optional, Tuple
 
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
     _get_token_base_cost,
+    _get_web_search_requests,
     _parse_prompt_tokens_details,
     calculate_cache_writing_cost,
     generic_cost_per_token,
@@ -110,11 +111,12 @@ def get_cost_for_anthropic_web_search(
     if model_info is None:
         return 0.0
 
-    if (
-        usage is None
-        or usage.server_tool_use is None
-        or usage.server_tool_use.web_search_requests is None
-    ):
+    if usage is None:
+        return 0.0
+    web_search_requests = _get_web_search_requests(
+        getattr(usage, "server_tool_use", None)
+    )
+    if web_search_requests is None:
         return 0.0
 
     ## Get the cost per web search request
@@ -128,5 +130,5 @@ def get_cost_for_anthropic_web_search(
         return 0.0
 
     ## Calculate the total cost
-    total_cost = cost_per_web_search_request * usage.server_tool_use.web_search_requests
+    total_cost = cost_per_web_search_request * web_search_requests
     return total_cost

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -1,5 +1,6 @@
 # What is this?
 ## Translates OpenAI call to Anthropic `/v1/messages` format
+import copy
 import json
 import traceback
 from collections import deque
@@ -29,6 +30,98 @@ if TYPE_CHECKING:
     from litellm.types.utils import ModelResponseStream
 
 
+class _CombinedChunkSplitter:
+    """
+    Splits a streaming chunk that carries BOTH response content and a
+    ``finish_reason`` into two chunks: a content-only chunk followed by a
+    finish-only chunk.
+
+    ``AnthropicStreamWrapper`` (via ``translate_streaming_openai_response_to_anthropic``)
+    assumes content and ``finish_reason`` never arrive in the same chunk — true for
+    real provider streams, but false for fake-streamed providers (e.g. Vertex AI
+    Gemma ``:predict``) where ``MockResponseIterator`` collapses the entire response
+    into a single chunk. Without this split the assumption causes all content to be
+    silently dropped (only the ``message_delta`` stop event is emitted).
+
+    Supports both sync and async iteration, since ``AnthropicStreamWrapper`` exposes
+    both ``__next__`` and ``__anext__``. An instance is single-mode: callers must
+    iterate it either synchronously or asynchronously, never both — the two modes
+    hold independent iterator references on the upstream stream and mixing them
+    would advance them out of sync.
+    """
+
+    def __init__(self, completion_stream: Any):
+        self._stream = completion_stream
+        self._sync_iter: Optional[Iterator[Any]] = None
+        self._async_iter: Optional[AsyncIterator[Any]] = None
+        self._buffer: deque = deque()
+
+    @staticmethod
+    def _is_combined(chunk: Any) -> bool:
+        """True if ``chunk`` carries response content AND a finish_reason."""
+        choices = getattr(chunk, "choices", None)
+        if not choices:
+            return False
+        choice = choices[0]
+        if getattr(choice, "finish_reason", None) is None:
+            return False
+        delta = getattr(choice, "delta", None)
+        if delta is None:
+            return False
+        return bool(
+            getattr(delta, "content", None)
+            or getattr(delta, "tool_calls", None)
+            or getattr(delta, "reasoning_content", None)
+            or getattr(delta, "thinking_blocks", None)
+        )
+
+    @staticmethod
+    def _split(chunk: Any) -> List[Any]:
+        """Return ``[chunk]``, or ``[content_chunk, finish_chunk]`` if combined."""
+        if not _CombinedChunkSplitter._is_combined(chunk):
+            return [chunk]
+
+        # Content chunk: keep the delta payload, clear the finish_reason.
+        content_chunk = copy.deepcopy(chunk)
+        content_chunk.choices[0].finish_reason = None
+
+        # Finish chunk: keep finish_reason (and usage), clear the delta payload.
+        finish_chunk = copy.deepcopy(chunk)
+        finish_delta = finish_chunk.choices[0].delta
+        finish_delta.content = None
+        if hasattr(finish_delta, "tool_calls"):
+            finish_delta.tool_calls = None
+        if hasattr(finish_delta, "reasoning_content"):
+            finish_delta.reasoning_content = None
+        if hasattr(finish_delta, "thinking_blocks"):
+            finish_delta.thinking_blocks = None
+        return [content_chunk, finish_chunk]
+
+    def __iter__(self) -> "Iterator[Any]":
+        return self
+
+    def __next__(self) -> Any:
+        if self._buffer:
+            return self._buffer.popleft()
+        if self._sync_iter is None:
+            self._sync_iter = iter(self._stream)
+        chunk = next(self._sync_iter)  # propagates StopIteration when exhausted
+        self._buffer.extend(self._split(chunk))
+        return self._buffer.popleft()
+
+    def __aiter__(self) -> "AsyncIterator[Any]":
+        return self
+
+    async def __anext__(self) -> Any:
+        if self._buffer:
+            return self._buffer.popleft()
+        if self._async_iter is None:
+            self._async_iter = self._stream.__aiter__()
+        chunk = await self._async_iter.__anext__()  # propagates StopAsyncIteration
+        self._buffer.extend(self._split(chunk))
+        return self._buffer.popleft()
+
+
 class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     """
     - first chunk return 'message_start'
@@ -62,7 +155,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         compaction_block: Optional[CompactionBlock] = None,
         iterations_usage: Optional[List[UsageIteration]] = None,
     ):
-        super().__init__(completion_stream)
+        # Wrap the upstream stream so chunks that carry both content and a
+        # finish_reason (fake-streamed providers) are split into two — see
+        # _CombinedChunkSplitter.
+        super().__init__(_CombinedChunkSplitter(completion_stream))
         self.model = model
         # Mapping of truncated tool names to original names (for OpenAI's 64-char limit)
         self.tool_name_mapping = tool_name_mapping or {}

litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py

@@ -155,10 +155,24 @@ class AnthropicResponsesStreamWrapper:
                 event.get("delta", "") if isinstance(event, dict) else ""
             )
             block_idx = (
-                self._item_id_to_block_index.get(item_id, self._current_block_index)
+                self._item_id_to_block_index.get(item_id, -1)
                 if item_id
                 else self._current_block_index
             )
+            if block_idx < 0:
+                # Some providers (e.g. LMStudio) skip response.output_item.added,
+                # so no text block is open yet; synthesize content_block_start
+                # instead of emitting a delta with index -1
+                block_idx = self._next_block_index()
+                if item_id:
+                    self._item_id_to_block_index[item_id] = block_idx
+                self._chunk_queue.append(
+                    {
+                        "type": "content_block_start",
+                        "index": block_idx,
+                        "content_block": {"type": "text", "text": ""},
+                    }
+                )
             self._chunk_queue.append(
                 {
                     "type": "content_block_delta",

litellm/llms/base_llm/base_model_iterator.py

@@ -50,6 +50,11 @@ def convert_model_response_to_streaming(
             model=model_response.model,
             choices=streaming_choices,
         )
+        # Carry usage onto the streaming chunk so fake-streamed responses
+        # (e.g. Vertex AI Gemma :predict) still report token counts.
+        usage = getattr(model_response, "usage", None)
+        if usage is not None:
+            setattr(processed_chunk, "usage", usage)
         return processed_chunk
     except Exception as e:
         raise ValueError(

litellm/llms/bedrock/chat/converse_transformation.py

@@ -920,10 +920,15 @@ class AmazonConverseConfig(BaseConfig):
                         continue
                     value = [value]
                 optional_params["stopSequences"] = value
-            if param == "temperature":
-                optional_params["temperature"] = value
-            if param == "top_p":
-                optional_params["topP"] = value
+            if param == "temperature" or param == "top_p":
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=optional_params,
+                    model=model,
+                    param=param,
+                    value=value,
+                    drop_params=drop_params,
+                    output_key="topP" if param == "top_p" else param,
+                )
             if param == "tools" and isinstance(value, list):
                 self._apply_tool_call_transformation(
                     tools=cast(List[OpenAIChatCompletionToolParam], value),
@@ -1221,7 +1226,9 @@ class AmazonConverseConfig(BaseConfig):
             inference_params["topK"] = inference_params.pop("top_k")
         return InferenceConfig(**inference_params)
 
-    def _handle_top_k_value(self, model: str, inference_params: dict) -> dict:
+    def _handle_top_k_value(
+        self, model: str, inference_params: dict, drop_params: bool = False
+    ) -> dict:
         base_model = BedrockModelInfo.get_base_model(model)
 
         val_top_k = None
@@ -1230,16 +1237,25 @@ class AmazonConverseConfig(BaseConfig):
         elif "top_k" in inference_params:
             val_top_k = inference_params.pop("top_k")
 
-        if val_top_k:
+        if val_top_k is not None:
             if base_model.startswith("anthropic"):
-                return {"top_k": val_top_k}
+                top_k_params: dict = {}
+                AnthropicConfig._apply_sampling_param(
+                    optional_params=top_k_params,
+                    model=model,
+                    param="top_k",
+                    value=val_top_k,
+                    drop_params=drop_params,
+                    output_key="top_k",
+                )
+                return top_k_params
             if base_model.startswith("amazon.nova"):
                 return {"inferenceConfig": {"topK": val_top_k}}
 
         return {}
 
     def _prepare_request_params(
-        self, optional_params: dict, model: str
+        self, optional_params: dict, model: str, drop_params: bool = False
     ) -> Tuple[dict, dict, dict, Optional[OutputConfigBlock]]:
         """Prepare and separate request parameters."""
         # Consume the internal ``_output_config_normalized`` marker set by
@@ -1338,7 +1354,7 @@ class AmazonConverseConfig(BaseConfig):
 
         # Only set the topK value in for models that support it
         additional_request_params.update(
-            self._handle_top_k_value(model, inference_params)
+            self._handle_top_k_value(model, inference_params, drop_params)
         )
 
         # Filter out internal/MCP-related parameters that shouldn't be sent to the API
@@ -1572,6 +1588,7 @@ class AmazonConverseConfig(BaseConfig):
         optional_params: dict,
         messages: Optional[List[AllMessageValues]] = None,
         headers: Optional[dict] = None,
+        drop_params: bool = False,
     ) -> CommonRequestObject:
         ## VALIDATE REQUEST
         """
@@ -1618,7 +1635,7 @@ class AmazonConverseConfig(BaseConfig):
             additional_request_params,
             request_metadata,
             output_config,
-        ) = self._prepare_request_params(optional_params, model)
+        ) = self._prepare_request_params(optional_params, model, drop_params)
 
         original_tools = inference_params.pop("tools", [])
 
@@ -1701,6 +1718,7 @@ class AmazonConverseConfig(BaseConfig):
             optional_params=optional_params,
             messages=messages,
             headers=headers,
+            drop_params=litellm_params.get("drop_params") is True,
         )
 
         bedrock_messages = (
@@ -1758,6 +1776,7 @@ class AmazonConverseConfig(BaseConfig):
             optional_params=optional_params,
             messages=messages,
             headers=headers,
+            drop_params=litellm_params.get("drop_params") is True,
         )
 
         ## TRANSFORMATION ##

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -48,6 +48,30 @@ class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
         region = self._get_aws_region_name(optional_params=optional_params, model=model)
         return MANTLE_ENDPOINT_TEMPLATE.format(region=region)
 
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        headers = super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            api_base=api_base,
+        )
+        project_id = litellm_params.get("aws_bedrock_project_id")
+        if project_id:
+            headers["anthropic-workspace"] = project_id
+        return headers
+
     def transform_request(
         self,
         model: str,

litellm/llms/bedrock/count_tokens/transformation.py

@@ -11,6 +11,11 @@ from typing import Any, Dict, List, Optional
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 
+# Placeholder satisfying the Anthropic InvokeModel schema's required
+# max_tokens field; CountTokens only counts input, so it has no effect
+# on any generation.
+DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS = 1024
+
 
 class BedrockCountTokensConfig(BaseAWSLLM):
     """
@@ -32,8 +37,20 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         Returns:
             'converse' or 'invokeModel'
         """
-        # If the request has messages in the expected Anthropic format, use converse
-        if "messages" in request_data and isinstance(request_data["messages"], list):
+        messages = request_data.get("messages")
+        if isinstance(messages, list):
+            # Anthropic content blocks carry a "type" key ({"type": "text", ...});
+            # Converse blocks don't ({"text": ...}, {"toolUse": ...}). Converse
+            # rejects Anthropic-shape blocks, so route those to invokeModel,
+            # which forwards the body verbatim.
+            for message in messages:
+                if not isinstance(message, dict):
+                    continue
+                content = message.get("content")
+                if isinstance(content, list) and any(
+                    isinstance(block, dict) and "type" in block for block in content
+                ):
+                    return "invokeModel"
             return "converse"
 
         # For raw text or other formats, use invokeModel
@@ -68,7 +85,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         {
             "input": {
                 "invokeModel": {
-                    "body": "{...raw model input...}"
+                    "body": "<base64-encoded raw model input>"
                 }
             }
         }
@@ -168,13 +185,24 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         self, request_data: Dict[str, Any]
     ) -> Dict[str, Any]:
         """Transform to InvokeModel input format."""
+        import base64
         import json
 
         # For InvokeModel, we need to provide the raw body that would be sent to the model
         # Remove the 'model' field from the body as it's not part of the model input
         body_data = {k: v for k, v in request_data.items() if k != "model"}
 
-        return {"input": {"invokeModel": {"body": json.dumps(body_data)}}}
+        if "messages" in body_data:
+            # Bedrock validates the body against the model's InvokeModel schema;
+            # Anthropic Messages bodies require these fields.
+            body_data.setdefault("anthropic_version", "bedrock-2023-05-31")
+            body_data.setdefault(
+                "max_tokens", DEFAULT_ANTHROPIC_INVOKE_MODEL_MAX_TOKENS
+            )
+
+        # The CountTokens API expects invokeModel.body as a base64-encoded blob
+        encoded_body = base64.b64encode(json.dumps(body_data).encode()).decode()
+        return {"input": {"invokeModel": {"body": encoded_body}}}
 
     def get_bedrock_count_tokens_endpoint(
         self,

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -6,7 +6,7 @@ AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix
 stripping that are specific to the bedrock-mantle endpoint.
 """
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
     AmazonAnthropicClaudeMessagesConfig,
@@ -45,6 +45,30 @@ class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
         region = self._get_aws_region_name(optional_params=optional_params, model=model)
         return MANTLE_ENDPOINT_TEMPLATE.format(region=region)
 
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> Tuple[dict, Optional[str]]:
+        headers, api_base = super().validate_anthropic_messages_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            api_base=api_base,
+        )
+        project_id = litellm_params.get("aws_bedrock_project_id")
+        if project_id:
+            headers["anthropic-workspace"] = project_id
+        return headers, api_base
+
     def transform_anthropic_messages_request(
         self,
         model: str,

litellm/llms/bedrock_mantle/chat/transformation.py

@@ -8,11 +8,12 @@ Auth: AWS Bedrock API key as Bearer token (set via BEDROCK_MANTLE_API_KEY env va
       or region-aware key via BEDROCK_MANTLE_{REGION}_API_KEY.
 """
 
-from typing import Iterator, AsyncIterator, Any, Optional, Tuple, Union
+from typing import Iterator, AsyncIterator, Any, List, Optional, Tuple, Union
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import AllMessageValues
 
 from ...openai_like.chat.transformation import OpenAILikeChatConfig
 
@@ -48,6 +49,30 @@ class BedrockMantleChatConfig(OpenAILikeChatConfig):
         dynamic_api_key = api_key or get_secret_str("BEDROCK_MANTLE_API_KEY")
         return api_base, dynamic_api_key
 
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        headers = super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=api_key,
+            api_base=api_base,
+        )
+        project_id = litellm_params.get("aws_bedrock_project_id")
+        if project_id:
+            headers["OpenAI-Project"] = project_id
+        return headers
+
     def get_supported_openai_params(self, model: str) -> list:
         base_params = super().get_supported_openai_params(model)
         try:

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -1,8 +1,10 @@
 """
 Amazon Bedrock Mantle - Responses API backend.
 
-gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses`
-path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI
+Mantle serves Responses on two upstream paths: gpt frontier models (gpt-5.5 /
+gpt-5.4) on `/openai/v1/responses`, and everything else that supports Responses
+(e.g. gpt-oss) on the standard `/v1/responses`. The gate picks the path per
+model and injects it via `use_openai_path`. Payloads and SSE follow the OpenAI
 Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides
 only the endpoint URL and authentication.
 
@@ -48,9 +50,14 @@ _MANTLE_HOST_RE = re.compile(
 
 
 class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
-    def __init__(self, aws_signer: Optional[BaseAWSLLM] = None):
+    def __init__(
+        self,
+        aws_signer: Optional[BaseAWSLLM] = None,
+        use_openai_path: bool = True,
+    ):
         super().__init__()
         self._aws_signer = aws_signer or BaseAWSLLM()
+        self.use_openai_path = use_openai_path
 
     @property
     def custom_llm_provider(self) -> LlmProviders:
@@ -94,7 +101,8 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
         # single resolved region so aws_region_name wins; preserve custom proxy hosts.
         if _MANTLE_HOST_RE.match(base):
             base = f"https://bedrock-mantle.{region}.api.aws"
-        return f"{base}/openai/v1/responses"
+        path = "/openai/v1/responses" if self.use_openai_path else "/v1/responses"
+        return f"{base}{path}"
 
     def validate_environment(
         self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams]
@@ -107,6 +115,8 @@ class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig):
         )
         if api_key:
             headers["Authorization"] = f"Bearer {api_key}"
+        if litellm_params.aws_bedrock_project_id:
+            headers["OpenAI-Project"] = litellm_params.aws_bedrock_project_id
         return headers
 
     def supports_native_file_search(self) -> bool:

litellm/llms/databricks/streaming_utils.py

@@ -25,6 +25,28 @@ class ModelResponseIterator:
             finish_reason = ""
             usage: Optional[ChatCompletionUsageBlock] = None
 
+            # Usage-only final chunk (OpenAI ``stream_options.include_usage``)
+            # arrives with an empty ``choices`` list — return usage without
+            # indexing ``choices[0]``.
+            if len(processed_chunk.choices) == 0:
+                final_usage = getattr(processed_chunk, "usage", None)
+                return GenericStreamingChunk(
+                    text="",
+                    tool_use=None,
+                    is_finished=False,
+                    finish_reason="",
+                    usage=(
+                        ChatCompletionUsageBlock(
+                            prompt_tokens=final_usage.prompt_tokens or 0,
+                            completion_tokens=final_usage.completion_tokens or 0,
+                            total_tokens=final_usage.total_tokens or 0,
+                        )
+                        if final_usage is not None
+                        else None
+                    ),
+                    index=0,
+                )
+
             if processed_chunk.choices[0].delta.content is not None:  # type: ignore
                 text = processed_chunk.choices[0].delta.content  # type: ignore
 

litellm/llms/litellm_proxy/skills/README.md

@@ -18,7 +18,7 @@ flowchart TB
         F[Request with container.skills] --> G[SkillsInjectionHook]
         G --> H{skill_id prefix?}
         
-        H -->|"litellm:skill_abc"| I[Fetch from LiteLLM DB]
+        H -->|"litellm_skill_abc"| I[Fetch from LiteLLM DB]
         H -->|"skill_xyz" no prefix| J[Pass to Anthropic as native skill]
         
         I --> K{Model provider?}
@@ -57,7 +57,7 @@ sequenceDiagram
     
     Note over LiteLLM,PreHook: PRE-CALL HOOK
     LiteLLM->>PreHook: Intercept request
-    PreHook->>PreHook: Fetch skill from DB (litellm:skill_id)
+    PreHook->>PreHook: Fetch skill from DB (litellm_skill_id)
     PreHook->>PreHook: Extract SKILL.md from ZIP
     PreHook->>PreHook: Inject SKILL.md into system prompt
     PreHook->>PreHook: Add litellm_code_execution tool
@@ -105,7 +105,7 @@ response = await litellm.acompletion(
     model="gpt-4o-mini",
     messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
     container={
-        "skills": [{"type": "custom", "skill_id": "litellm:skill_abc123"}]
+        "skills": [{"type": "custom", "skill_id": "litellm_skill_abc123"}]
     },
 )
 
@@ -261,7 +261,7 @@ response = litellm.completion(
     messages=[{"role": "user", "content": "Analyze this data..."}],
     container={
         "skills": [
-            {"type": "custom", "skill_id": "litellm:skill_abc123"}  # litellm: prefix
+            {"type": "custom", "skill_id": "litellm_skill_abc123"}  # litellm_skill_ prefix
         ]
     }
 )
@@ -277,7 +277,7 @@ response = litellm.completion(
     "messages": [{"role": "user", "content": "Help me analyze data"}],
     "container": {
         "skills": [
-            {"type": "custom", "skill_id": "litellm:skill_abc123"}
+            {"type": "custom", "skill_id": "litellm_skill_abc123"}
         ]
     }
 }
@@ -287,7 +287,7 @@ response = litellm.completion(
 
 The hook (`litellm/proxy/hooks/litellm_skills/main.py`) intercepts the request:
 
-1. **Detects `litellm:` prefix** → Fetches skill from database
+1. **Detects `litellm_skill_` prefix** → Fetches skill from database
 2. **Checks model provider** → Bedrock is not Anthropic
 3. **Extracts SKILL.md** from stored ZIP file
 4. **Converts skill to tool** + **Injects content into system prompt**
@@ -361,8 +361,8 @@ model LiteLLM_SkillsTable {
 | Create skill on Anthropic | `anthropic` | N/A | Forward to Anthropic API |
 | Create skill in LiteLLM DB | `litellm_proxy` | N/A | Store in database |
 | Use Anthropic native skill | N/A | `skill_xyz` | Pass to Anthropic container.skills |
-| Use LiteLLM skill on Anthropic | N/A | `litellm:skill_abc` | Convert to tools |
-| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm:skill_abc` | Convert to tools + inject SKILL.md |
+| Use LiteLLM skill on Anthropic | N/A | `litellm_skill_abc` | Convert to tools |
+| Use LiteLLM skill on Bedrock/OpenAI | N/A | `litellm_skill_abc` | Convert to tools + inject SKILL.md |
 
 ## Testing
 

litellm/llms/litellm_proxy/skills/constants.py

@@ -4,6 +4,10 @@ Constants for LiteLLM Skills
 Centralized constants for skills processing, code execution, and sandbox configuration.
 """
 
+LITELLM_SKILL_ID_PREFIX: str = "litellm_skill_"
+"""Prefix for DB-backed skill IDs. The model-facing tool name is the skill ID
+with hyphens/spaces replaced by underscores, which leaves this prefix intact."""
+
 # Code execution loop settings
 DEFAULT_MAX_ITERATIONS: int = 10
 """Maximum number of iterations for the automatic code execution loop."""

litellm/llms/litellm_proxy/skills/handler.py

@@ -10,6 +10,7 @@ from typing import Any, Dict, List, Optional
 
 from litellm._logging import verbose_logger
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.llms.litellm_proxy.skills.constants import LITELLM_SKILL_ID_PREFIX
 from litellm.proxy._types import LiteLLM_SkillsTable, NewSkillRequest, UserAPIKeyAuth
 from litellm.proxy.common_utils.resource_ownership import (
     get_primary_resource_owner_scope,
@@ -68,7 +69,7 @@ class LiteLLMSkillsHandler:
     ) -> LiteLLM_SkillsTable:
         prisma_client = await LiteLLMSkillsHandler._get_prisma_client()
 
-        skill_id = f"litellm_skill_{uuid.uuid4()}"
+        skill_id = f"{LITELLM_SKILL_ID_PREFIX}{uuid.uuid4()}"
         owner = get_primary_resource_owner_scope(user_api_key_dict) or user_id
         if owner is None:
             # Identity-less callers (no user_id / team_id / org_id /

litellm/llms/openai/completion/handler.py

@@ -49,6 +49,8 @@ class OpenAITextCompletion(BaseLLM):
         headers: Optional[dict] = None,
     ):
         try:
+            if headers:
+                optional_params = {**optional_params, "extra_headers": headers}
             if headers is None:
                 headers = self.validate_environment(api_key=api_key)
             if model is None or messages is None:

litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py

@@ -19,7 +19,7 @@ from litellm.types.llms.vertex_ai import (
     VertexAICachedContentResponseObject,
 )
 
-from ..common_utils import VertexAIError
+from ..common_utils import VertexAIError, get_vertex_base_url
 from ..vertex_llm_base import VertexBase
 from .transformation import (
     separate_cached_messages,
@@ -69,17 +69,13 @@ class ContextCachingEndpoints(VertexBase):
         elif custom_llm_provider == "vertex_ai":
             auth_header = vertex_auth_header
             endpoint = "cachedContents"
-            if vertex_location == "global":
-                url = f"https://aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
-            else:
-                url = f"https://{vertex_location}-aiplatform.googleapis.com/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
+            base_url = get_vertex_base_url(vertex_location)
+            url = f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
         else:
             auth_header = vertex_auth_header
             endpoint = "cachedContents"
-            if vertex_location == "global":
-                url = f"https://aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
-            else:
-                url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
+            base_url = get_vertex_base_url(vertex_location)
+            url = f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/{endpoint}"
 
         return self._check_custom_proxy(
             api_base=api_base,

litellm/llms/xai/chat/transformation.py

@@ -5,6 +5,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
+from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     filter_value_from_dict,
     strip_name_from_messages,
@@ -39,6 +40,72 @@ class XAIChatConfig(OpenAIGPTConfig):
         dynamic_api_key = XAIModelInfo.get_api_key(api_key)
         return api_base, dynamic_api_key
 
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        from litellm.llms.xai.oauth import (
+            XAIOAuthAuthenticator,
+            XAIOAuthError,
+            should_use_xai_oauth,
+        )
+
+        dynamic_api_key = XAIModelInfo.get_api_key(api_key)
+        if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
+            try:
+                headers["Authorization"] = (
+                    f"Bearer {XAIOAuthAuthenticator().get_access_token()}"
+                )
+            except XAIOAuthError as exc:
+                raise AuthenticationError(
+                    model=model,
+                    llm_provider=self.custom_llm_provider or "xai",
+                    message=str(exc),
+                ) from exc
+            if "content-type" not in headers and "Content-Type" not in headers:
+                headers["Content-Type"] = "application/json"
+            return headers
+
+        return super().validate_environment(
+            headers=headers,
+            model=model,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            api_key=dynamic_api_key,
+            api_base=api_base,
+        )
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
+
+        dynamic_api_key = XAIModelInfo.get_api_key(api_key)
+        if should_use_xai_oauth(litellm_params) and not dynamic_api_key:
+            api_base = XAIOAuthAuthenticator().get_api_base()
+
+        return super().get_complete_url(
+            api_base=api_base,
+            api_key=dynamic_api_key,
+            model=model,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            stream=stream,
+        )
+
     def get_supported_openai_params(self, model: str) -> list:
         base_openai_params = [
             "logit_bias",

litellm/llms/xai/oauth.py

@@ -0,0 +1,421 @@
+import base64
+import hashlib
+import json
+import os
+import secrets
+import sys
+import threading
+import time
+import uuid
+import webbrowser
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Any, Dict, Optional, Tuple, Union
+from urllib.parse import parse_qs, urlencode, urlparse
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.constants import XAI_API_BASE
+from litellm.llms.custom_httpx.http_handler import HTTPHandler, _get_httpx_client
+from litellm.secret_managers.main import get_secret_str
+
+XAI_OAUTH_ISSUER = "https://auth.x.ai"
+XAI_OAUTH_DISCOVERY_URL = f"{XAI_OAUTH_ISSUER}/.well-known/openid-configuration"
+XAI_OAUTH_CLIENT_ID = "b1a00492-073a-47ea-816f-4c329264a828"
+XAI_OAUTH_SCOPE = "openid profile email offline_access grok-cli:access api:access"
+XAI_OAUTH_REDIRECT_HOST = "127.0.0.1"
+XAI_OAUTH_REDIRECT_PORT = 56121
+XAI_OAUTH_REDIRECT_PATH = "/callback"
+XAI_OAUTH_EXPIRY_SKEW_SECONDS = 120
+XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS = 180
+_XAI_OAUTH_REFRESH_LOCK = threading.Lock()
+
+
+class XAIOAuthError(Exception):
+    pass
+
+
+class XAIOAuthLoginRequiredError(XAIOAuthError):
+    pass
+
+
+class _CallbackHandler(BaseHTTPRequestHandler):
+    server: "_CallbackServer"
+
+    def do_GET(self) -> None:
+        parsed = urlparse(self.path)
+        if parsed.path != XAI_OAUTH_REDIRECT_PATH:
+            self.send_response(404)
+            self.end_headers()
+            return
+
+        params = parse_qs(parsed.query)
+        result = {
+            "code": params.get("code", [None])[0],
+            "state": params.get("state", [None])[0],
+            "error": params.get("error", [None])[0],
+            "error_description": params.get("error_description", [None])[0],
+        }
+        self.server.callback_result = result
+
+        if result["state"] != self.server.expected_state:
+            self.send_response(400)
+            self.send_header("Content-Type", "text/html; charset=utf-8")
+            self.end_headers()
+            self.wfile.write(
+                b"<html><body><h1>xAI authorization state mismatch.</h1></body></html>"
+            )
+            return
+
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.end_headers()
+        body = (
+            b"<html><body><h1>xAI authorization failed.</h1>You can close this tab.</body></html>"
+            if result["error"]
+            else b"<html><body><h1>xAI authorization received.</h1>You can close this tab.</body></html>"
+        )
+        self.wfile.write(body)
+
+    def log_message(self, format: str, *args: Any) -> None:
+        return
+
+
+class _CallbackServer(HTTPServer):
+    expected_state: str
+    callback_result: Optional[Dict[str, Optional[str]]]
+
+
+class XAIOAuthAuthenticator:
+    def __init__(
+        self, http_client: Optional[Union[httpx.Client, HTTPHandler]] = None
+    ) -> None:
+        self.token_dir = get_secret_str("XAI_OAUTH_TOKEN_DIR") or os.path.expanduser(
+            "~/.config/litellm/xai_oauth"
+        )
+        self.auth_file = os.path.join(
+            self.token_dir, get_secret_str("XAI_OAUTH_AUTH_FILE") or "auth.json"
+        )
+        self.http_client = http_client
+
+    def get_api_base(self) -> str:
+        return (
+            get_secret_str("XAI_OAUTH_API_BASE")
+            or get_secret_str("XAI_API_BASE")
+            or XAI_API_BASE
+        )
+
+    def get_access_token(self) -> str:
+        auth_data = self._read_auth_file()
+        if not auth_data:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth login required. Run `litellm xai-oauth login`."
+            )
+
+        access_token = auth_data.get("access_token")
+        if access_token and not self._is_expired(auth_data):
+            return access_token
+
+        refresh_token = auth_data.get("refresh_token")
+        if not refresh_token:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
+            )
+
+        with _XAI_OAUTH_REFRESH_LOCK:
+            locked_auth_data = self._read_auth_file() or auth_data
+            access_token = locked_auth_data.get("access_token")
+            if access_token and not self._is_expired(locked_auth_data):
+                return access_token
+
+            refreshed = self._refresh_tokens(locked_auth_data)
+            return refreshed["access_token"]
+
+    def login(self, force: bool = False, no_browser: bool = False) -> Dict[str, Any]:
+        existing = self._read_auth_file()
+        if existing and not force and existing.get("access_token"):
+            if not self._is_expired(existing):
+                return existing
+            if existing.get("refresh_token"):
+                try:
+                    return self._refresh_tokens(existing)
+                except XAIOAuthError:
+                    pass
+
+        discovery = self._discover()
+        verifier, challenge = self._pkce_pair()
+        state = uuid.uuid4().hex
+        nonce = uuid.uuid4().hex
+        server, redirect_uri = self._start_callback_server(state)
+        authorize_url = self._build_authorize_url(
+            authorization_endpoint=discovery["authorization_endpoint"],
+            redirect_uri=redirect_uri,
+            challenge=challenge,
+            state=state,
+            nonce=nonce,
+        )
+
+        if no_browser or not webbrowser.open(authorize_url):
+            sys.stdout.write(
+                f"Open this URL to authenticate with xAI:\n{authorize_url}\n"
+            )
+            sys.stdout.flush()
+
+        result = self._wait_for_callback(server)
+        if result.get("state") != state:
+            raise XAIOAuthError("xAI OAuth state mismatch")
+        if result.get("error"):
+            description = result.get("error_description") or result["error"]
+            raise XAIOAuthError(f"xAI authorization failed: {description}")
+        code = result.get("code")
+        if not code:
+            raise XAIOAuthError("xAI authorization failed: no code returned")
+
+        token_payload = self._exchange_token(
+            discovery["token_endpoint"],
+            {
+                "grant_type": "authorization_code",
+                "code": code,
+                "redirect_uri": redirect_uri,
+                "client_id": XAI_OAUTH_CLIENT_ID,
+                "code_verifier": verifier,
+            },
+        )
+        auth_data = self._build_auth_record(token_payload, discovery["token_endpoint"])
+        self._write_auth_file(auth_data)
+        return auth_data
+
+    def _client(self) -> Union[httpx.Client, HTTPHandler]:
+        return self.http_client or _get_httpx_client()
+
+    def _ensure_token_dir(self) -> None:
+        os.makedirs(self.token_dir, mode=0o700, exist_ok=True)
+        try:
+            os.chmod(self.token_dir, 0o700)
+        except OSError:
+            verbose_logger.debug("Could not chmod xAI OAuth token directory")
+
+    def _read_auth_file(self) -> Optional[Dict[str, Any]]:
+        try:
+            with open(self.auth_file, "r") as f:
+                data = json.load(f)
+            return data if isinstance(data, dict) else None
+        except (IOError, json.JSONDecodeError):
+            return None
+
+    def _write_auth_file(self, data: Dict[str, Any]) -> None:
+        self._ensure_token_dir()
+        tmp_file = os.path.join(
+            self.token_dir,
+            f".{os.path.basename(self.auth_file)}.{uuid.uuid4().hex}.tmp",
+        )
+        flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL
+        if hasattr(os, "O_NOFOLLOW"):
+            flags |= os.O_NOFOLLOW
+        fd = os.open(tmp_file, flags, 0o600)
+        try:
+            with os.fdopen(fd, "w") as f:
+                json.dump(data, f)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(tmp_file, self.auth_file)
+            try:
+                os.chmod(self.auth_file, 0o600)
+            except OSError:
+                verbose_logger.debug("Could not chmod xAI OAuth auth file")
+        except Exception:
+            try:
+                os.close(fd)
+            except OSError:
+                pass
+            try:
+                os.unlink(tmp_file)
+            except OSError:
+                pass
+            raise
+
+    def _is_expired(self, auth_data: Dict[str, Any]) -> bool:
+        expires_at = auth_data.get("expires_at")
+        if expires_at is None:
+            return True
+        try:
+            return time.time() >= float(expires_at) - XAI_OAUTH_EXPIRY_SKEW_SECONDS
+        except (TypeError, ValueError):
+            return True
+
+    def _discover(self) -> Dict[str, str]:
+        try:
+            response = self._client().get(
+                XAI_OAUTH_DISCOVERY_URL, headers={"Accept": "application/json"}
+            )
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise XAIOAuthError(
+                f"xAI OAuth discovery request failed: {exc.response.status_code} {exc.response.text}"
+            ) from exc
+        try:
+            data = response.json()
+        except ValueError as exc:
+            raise XAIOAuthError(
+                "xAI OAuth discovery response was not valid JSON"
+            ) from exc
+        authorization_endpoint = data.get("authorization_endpoint")
+        token_endpoint = data.get("token_endpoint")
+        if not authorization_endpoint or not token_endpoint:
+            raise XAIOAuthError("xAI OAuth discovery missing endpoints")
+        return {
+            "authorization_endpoint": self._validate_xai_endpoint(
+                authorization_endpoint
+            ),
+            "token_endpoint": self._validate_xai_endpoint(token_endpoint),
+        }
+
+    def _validate_xai_endpoint(self, url: str) -> str:
+        parsed = urlparse(url)
+        host = (parsed.hostname or "").lower()
+        if parsed.scheme != "https" or (host != "x.ai" and not host.endswith(".x.ai")):
+            raise XAIOAuthError(
+                f"xAI OAuth discovery returned unexpected endpoint: {url}"
+            )
+        return url
+
+    def _pkce_pair(self) -> Tuple[str, str]:
+        verifier = (
+            base64.urlsafe_b64encode(secrets.token_bytes(32)).rstrip(b"=").decode()
+        )
+        challenge = (
+            base64.urlsafe_b64encode(hashlib.sha256(verifier.encode()).digest())
+            .rstrip(b"=")
+            .decode()
+        )
+        return verifier, challenge
+
+    def _start_callback_server(self, state: str) -> Tuple[_CallbackServer, str]:
+        last_error: Optional[OSError] = None
+        for port in (XAI_OAUTH_REDIRECT_PORT, 0):
+            try:
+                server = _CallbackServer(
+                    (XAI_OAUTH_REDIRECT_HOST, port), _CallbackHandler
+                )
+                server.expected_state = state
+                server.callback_result = None
+                actual_port = server.server_address[1]
+                redirect_uri = f"http://{XAI_OAUTH_REDIRECT_HOST}:{actual_port}{XAI_OAUTH_REDIRECT_PATH}"
+                return server, redirect_uri
+            except OSError as exc:
+                last_error = exc
+        raise XAIOAuthError(f"Could not start xAI OAuth callback server: {last_error}")
+
+    def _build_authorize_url(
+        self,
+        authorization_endpoint: str,
+        redirect_uri: str,
+        challenge: str,
+        state: str,
+        nonce: str,
+    ) -> str:
+        params = {
+            "response_type": "code",
+            "client_id": XAI_OAUTH_CLIENT_ID,
+            "redirect_uri": redirect_uri,
+            "scope": XAI_OAUTH_SCOPE,
+            "code_challenge": challenge,
+            "code_challenge_method": "S256",
+            "state": state,
+            "nonce": nonce,
+        }
+        return f"{authorization_endpoint}?{urlencode(params)}"
+
+    def _wait_for_callback(self, server: _CallbackServer) -> Dict[str, Optional[str]]:
+        server.timeout = 1
+        deadline = time.time() + XAI_OAUTH_CALLBACK_TIMEOUT_SECONDS
+        try:
+            while time.time() < deadline:
+                server.handle_request()
+                if server.callback_result is not None:
+                    return server.callback_result
+        finally:
+            server.server_close()
+        raise XAIOAuthError("Timed out waiting for xAI OAuth callback")
+
+    def _exchange_token(
+        self, token_endpoint: str, data: Dict[str, str]
+    ) -> Dict[str, Any]:
+        try:
+            response = self._client().post(
+                token_endpoint,
+                headers={
+                    "Accept": "application/json",
+                    "Content-Type": "application/x-www-form-urlencoded",
+                },
+                data=data,
+            )
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            raise XAIOAuthError(
+                f"xAI OAuth token request failed: {exc.response.status_code} {exc.response.text}"
+            ) from exc
+        try:
+            body = response.json()
+        except ValueError as exc:
+            raise XAIOAuthError("xAI OAuth token response was not valid JSON") from exc
+        if not isinstance(body, dict):
+            raise XAIOAuthError("xAI OAuth token response was not an object")
+        return body
+
+    def _build_auth_record(
+        self,
+        token_payload: Dict[str, Any],
+        token_endpoint: str,
+        fallback_refresh_token: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        access_token = token_payload.get("access_token")
+        refresh_token = token_payload.get("refresh_token") or fallback_refresh_token
+        if not access_token:
+            raise XAIOAuthError("xAI OAuth token response missing access_token")
+        if not refresh_token:
+            raise XAIOAuthError("xAI OAuth token response missing refresh_token")
+        expires_in = token_payload.get("expires_in") or 3600
+        try:
+            expires_at = int(time.time() + int(expires_in))
+        except (TypeError, ValueError):
+            expires_at = int(time.time() + 3600)
+        return {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "id_token": token_payload.get("id_token"),
+            "token_type": token_payload.get("token_type") or "Bearer",
+            "token_endpoint": token_endpoint,
+            "expires_at": expires_at,
+        }
+
+    def _refresh_tokens(self, auth_data: Dict[str, Any]) -> Dict[str, Any]:
+        token_endpoint = auth_data.get("token_endpoint")
+        if not token_endpoint:
+            token_endpoint = self._discover()["token_endpoint"]
+        token_endpoint = self._validate_xai_endpoint(token_endpoint)
+        refresh_token = auth_data.get("refresh_token")
+        if not refresh_token:
+            raise XAIOAuthLoginRequiredError(
+                "xAI OAuth refresh token missing. Run `litellm xai-oauth login`."
+            )
+
+        token_payload = self._exchange_token(
+            token_endpoint,
+            {
+                "grant_type": "refresh_token",
+                "refresh_token": refresh_token,
+                "client_id": XAI_OAUTH_CLIENT_ID,
+            },
+        )
+        refreshed = self._build_auth_record(
+            token_payload,
+            token_endpoint,
+            fallback_refresh_token=refresh_token,
+        )
+        self._write_auth_file(refreshed)
+        return refreshed
+
+
+def should_use_xai_oauth(litellm_params: Optional[Dict[str, Any]]) -> bool:
+    return bool((litellm_params or {}).get("use_xai_oauth"))

litellm/llms/xai/responses/transformation.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import XAI_API_BASE
+from litellm.exceptions import AuthenticationError
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.llms.xai.common_utils import XAIModelInfo
 from litellm.secret_managers.main import get_secret_str
@@ -220,10 +221,27 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
             litellm_params.api_key, legacy_generic_before_env=True
         )
 
+        if not api_key:
+            from litellm.llms.xai.oauth import (
+                XAIOAuthAuthenticator,
+                XAIOAuthError,
+                should_use_xai_oauth,
+            )
+
+            if should_use_xai_oauth(litellm_params.model_dump()):
+                try:
+                    api_key = XAIOAuthAuthenticator().get_access_token()
+                except XAIOAuthError as exc:
+                    raise AuthenticationError(
+                        model=model,
+                        llm_provider=self.custom_llm_provider.value,
+                        message=str(exc),
+                    ) from exc
+
         if not api_key:
             raise ValueError(
                 "XAI API key is required. Set api_key, litellm.xai_key, "
-                "litellm.api_key, or XAI_API_KEY."
+                "litellm.api_key, XAI_API_KEY, or use_xai_oauth=True."
             )
 
         headers.update(
@@ -244,12 +262,20 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         Returns:
             str: The full URL for the XAI /responses endpoint
         """
-        api_base = (
-            api_base
-            or litellm.api_base
-            or get_secret_str("XAI_API_BASE")
-            or XAI_API_BASE
+        from litellm.llms.xai.oauth import XAIOAuthAuthenticator, should_use_xai_oauth
+
+        api_key = XAIModelInfo.get_api_key(
+            litellm_params.get("api_key"), legacy_generic_before_env=True
         )
+        if should_use_xai_oauth(litellm_params) and not api_key:
+            api_base = XAIOAuthAuthenticator().get_api_base()
+        else:
+            api_base = (
+                api_base
+                or litellm.api_base
+                or get_secret_str("XAI_API_BASE")
+                or XAI_API_BASE
+            )
 
         # Remove trailing slashes
         api_base = api_base.rstrip("/")

litellm/main.py

@@ -1638,6 +1638,8 @@ def completion(  # type: ignore # noqa: PLR0915
             litellm_request_debug=kwargs.get("litellm_request_debug", False),
             tpm=kwargs.get("tpm"),
             rpm=kwargs.get("rpm"),
+            use_xai_oauth=kwargs.get("use_xai_oauth", False),
+            aws_bedrock_project_id=kwargs.get("aws_bedrock_project_id"),
         )
         cast(LiteLLMLoggingObj, logging).update_environment_variables(
             model=model,
@@ -2134,9 +2136,6 @@ def completion(  # type: ignore # noqa: PLR0915
 
             headers = headers or litellm.headers
 
-            if extra_headers is not None:
-                optional_params["extra_headers"] = extra_headers
-
             ## LOAD CONFIG - if set
             config = litellm.OpenAITextCompletionConfig.get_config()
             for k, v in config.items():
@@ -2162,6 +2161,7 @@ def completion(  # type: ignore # noqa: PLR0915
             _response = openai_text_completions.completion(
                 model=model,
                 messages=messages,
+                headers=headers,
                 model_response=model_response,
                 print_verbose=print_verbose,
                 api_key=api_key,

litellm/model_prices_and_context_window_backup.json

@@ -1156,6 +1156,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "azure_ai/claude-fable-5": {
+        "input_cost_per_token": 1e-05,
+        "output_cost_per_token": 5e-05,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10170,6 +10345,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10204,6 +10380,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10214,6 +10391,40 @@
         },
         "supports_output_config": true
     },
+    "claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-opus-4-8": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10238,6 +10449,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -24180,9 +24392,12 @@
         "max_output_tokens": 8192
     },
     "minimax/MiniMax-M3": {
-        "input_cost_per_token": 6e-07,
-        "output_cost_per_token": 2.4e-06,
-        "cache_read_input_token_cost": 1.2e-07,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_above_512k_tokens": 6e-07,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_512k_tokens": 2.4e-06,
+        "cache_read_input_token_cost": 6e-08,
+        "cache_read_input_token_cost_above_512k_tokens": 1.2e-07,
         "litellm_provider": "minimax",
         "mode": "chat",
         "supports_function_calling": true,
@@ -24191,7 +24406,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_vision": true,
-        "max_input_tokens": 512000,
+        "max_input_tokens": 1000000,
         "max_output_tokens": 128000
     },
     "mistral.devstral-2-123b": {
@@ -34004,6 +34219,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34032,6 +34248,67 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5@default": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34061,6 +34338,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34090,6 +34368,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -41370,6 +41649,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41389,6 +41669,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41690,5 +41971,164 @@
             "/v1/audio/transcriptions"
         ],
         "supports_audio_input": true
+    },
+    "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 6e-07,
+        "output_cost_per_token": 3.6e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 4.5e-07,
+        "output_cost_per_token": 1.8e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/Qwen/Qwen3.6-27B-FP8": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 3.2e-07,
+        "output_cost_per_token": 3.2e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-06,
+        "output_cost_per_token": 4.4e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 202752,
+        "max_output_tokens": 202752,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/deepseek-ai/DeepSeek-V4-Flash": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-07,
+        "output_cost_per_token": 2.8e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/moonshotai/Kimi-K2.6": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 9.6e-07,
+        "output_cost_per_token": 4e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/MiniMaxAI/MiniMax-M2.5": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 1.2e-06,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 196608,
+        "max_output_tokens": 196608,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/google/gemma-4-31B-it": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.4e-07,
+        "output_cost_per_token": 5.6e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/openai/gpt-oss-120b": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 1.5e-07,
+        "output_cost_per_token": 6e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+    },
+    "tensormesh/openai/gpt-oss-20b": {
+        "litellm_provider": "tensormesh",
+        "mode": "chat",
+        "input_cost_per_token": 7e-08,
+        "output_cost_per_token": 2.8e-07,
+        "cache_read_input_token_cost": 0,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "supports_function_calling": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "supports_system_messages": true,
+        "supports_reasoning": true,
+        "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
     }
 }

litellm/proxy/_experimental/mcp_server/db.py

@@ -568,10 +568,12 @@ async def delete_mcp_server(
     """
     Delete the mcp server from the db by server_id
 
-    The server-row delete is the commit point. Per-user env var rows have no FK
-    cascade, so they are cleaned up afterwards on a best-effort basis: a transient
-    failure there leaves only orphaned rows pointing at a now-missing server and
-    must not turn a successful delete into a caller-visible error.
+    The server-row delete is the commit point. Per-user credential and env var
+    rows have no FK cascade, so they are cleaned up afterwards on a best-effort
+    basis: a transient failure there leaves only orphaned rows pointing at a
+    now-missing server and must not turn a successful delete into a
+    caller-visible error. Each table is cleaned independently so a failure on one
+    still attempts the other.
 
     Returns the deleted mcp server record if it exists, otherwise None
     """
@@ -581,17 +583,20 @@ async def delete_mcp_server(
         },
     )
     if deleted_server is not None:
-        try:
-            await prisma_client.db.litellm_mcpuserenvvars.delete_many(
-                where={"server_id": server_id}
-            )
-        except Exception as e:
-            verbose_proxy_logger.warning(
-                "MCP server %s deleted but per-user env var cleanup failed; "
-                "orphaned rows can be removed on a later delete: %s",
-                server_id,
-                e,
-            )
+        for model, label in (
+            (prisma_client.db.litellm_mcpusercredentials, "credential"),
+            (prisma_client.db.litellm_mcpuserenvvars, "env var"),
+        ):
+            try:
+                await model.delete_many(where={"server_id": server_id})
+            except Exception as e:
+                verbose_proxy_logger.warning(
+                    "MCP server %s deleted but per-user %s cleanup failed; "
+                    "orphaned rows can be removed on a later delete: %s",
+                    server_id,
+                    label,
+                    e,
+                )
     return deleted_server
 
 

litellm/proxy/_experimental/mcp_server/mcp_context.py

@@ -19,3 +19,9 @@ _mcp_active_toolset_id: ContextVar[Optional[str]] = ContextVar(
 _mcp_gateway_initialize_instructions: ContextVar[Optional[str]] = ContextVar(
     "_mcp_gateway_initialize_instructions", default=None
 )
+
+# Per-request scoped server name; set in MCP HTTP/SSE handlers when the path
+# identifies exactly one upstream server. Never populated from client-supplied headers.
+_mcp_gateway_server_name: ContextVar[Optional[str]] = ContextVar(
+    "_mcp_gateway_server_name", default=None
+)

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -354,6 +354,52 @@ def _deserialize_json_list(data: Any) -> Optional[List[Dict[str, Any]]]:
     ]
 
 
+def _normalize_mcp_server_cost_info(mcp_info: MCPInfo) -> None:
+    """Coerce ``mcp_server_cost_info`` numeric fields to ``float`` at ingest.
+
+    YAML 1.1 parses scientific notation without a decimal point (e.g.
+    ``7e-05``) as a string, and ``MCPServerCostInfo`` is a TypedDict with no
+    runtime validation, so string-typed costs flow through to the UI and
+    crash its ``.toFixed`` formatting. Values that cannot be coerced are
+    dropped with a warning instead of failing the server load.
+    """
+    cost_info = mcp_info.get("mcp_server_cost_info")
+    if not isinstance(cost_info, dict):
+        return
+
+    server_name = mcp_info.get("server_name")
+    normalized = dict(cost_info)
+
+    default_cost = normalized.get("default_cost_per_query")
+    if default_cost is not None:
+        try:
+            normalized["default_cost_per_query"] = float(default_cost)
+        except (TypeError, ValueError):
+            verbose_logger.warning(
+                "MCP server '%s' has non-numeric default_cost_per_query %r; ignoring it",
+                server_name,
+                default_cost,
+            )
+            del normalized["default_cost_per_query"]
+
+    tool_costs = normalized.get("tool_name_to_cost_per_query")
+    if isinstance(tool_costs, dict):
+        normalized_tool_costs = {}
+        for tool_name, cost in tool_costs.items():
+            try:
+                normalized_tool_costs[tool_name] = float(cost)
+            except (TypeError, ValueError):
+                verbose_logger.warning(
+                    "MCP server '%s' has non-numeric cost %r for tool '%s'; ignoring it",
+                    server_name,
+                    cost,
+                    tool_name,
+                )
+        normalized["tool_name_to_cost_per_query"] = normalized_tool_costs
+
+    mcp_info["mcp_server_cost_info"] = normalized
+
+
 def _create_sampling_callback(user_api_key_auth: Optional[Any] = None):
     """
     Create a sampling callback for MCP ClientSession.
@@ -621,6 +667,7 @@ class MCPServerManager:
                 mcp_info["server_name"] = server_name
             if "description" not in mcp_info and server_config.get("description"):
                 mcp_info["description"] = server_config.get("description")
+            _normalize_mcp_server_cost_info(mcp_info)
 
             # Use alias for name if present, else server_name
             alias = server_config.get("alias", None)
@@ -1091,6 +1138,7 @@ class MCPServerManager:
             mcp_info["server_name"] = mcp_server.server_name or mcp_server.server_id
         if "description" not in mcp_info and mcp_server.description:
             mcp_info["description"] = mcp_server.description
+        _normalize_mcp_server_cost_info(mcp_info)
 
         auth_type = cast(MCPAuthType, mcp_server.auth_type)
         server_url = mcp_server.url

litellm/proxy/_experimental/mcp_server/server.py

@@ -47,6 +47,7 @@ from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
 from litellm.proxy._experimental.mcp_server.mcp_context import (
     _mcp_active_toolset_id,
     _mcp_gateway_initialize_instructions,
+    _mcp_gateway_server_name,
 )
 from litellm.proxy._experimental.mcp_server.mcp_debug import MCPDebug
 from litellm.proxy._experimental.mcp_server.utils import (
@@ -323,10 +324,14 @@ if MCP_AVAILABLE:
             notification_options=notification_options,
             experimental_capabilities=experimental_capabilities or {},
         )
+        updates: Dict[str, Any] = {}
         merged = _mcp_gateway_initialize_instructions.get()
         if merged is not None:
-            return opts.model_copy(update={"instructions": merged})
-        return opts
+            updates["instructions"] = merged
+        scoped_server_name = _mcp_gateway_server_name.get()
+        if scoped_server_name is not None:
+            updates["server_name"] = scoped_server_name
+        return opts.model_copy(update=updates) if updates else opts
 
     ########################################################
     ############ Initialize the MCP Server #################
@@ -1544,6 +1549,7 @@ if MCP_AVAILABLE:
         user_api_key_auth: Optional[UserAPIKeyAuth],
         mcp_servers: Optional[List[str]],
         client_ip: Optional[str],
+        scoped_server_endpoint: bool = False,
     ) -> AsyncIterator[None]:
         allowed = await _get_allowed_mcp_servers(
             user_api_key_auth=user_api_key_auth,
@@ -1565,11 +1571,22 @@ if MCP_AVAILABLE:
                 return_exceptions=True,
             )
         merged = _merge_gateway_initialize_instructions(allowed_mcp_servers=allowed)
-        tok = _mcp_gateway_initialize_instructions.set(merged)
+        scoped_server_name = None
+        if scoped_server_endpoint and len(allowed) == 1:
+            scoped_server = allowed[0]
+            scoped_server_name = (
+                scoped_server.alias
+                or scoped_server.server_name
+                or scoped_server.name
+                or scoped_server.server_id
+            )
+        instructions_token = _mcp_gateway_initialize_instructions.set(merged)
+        server_name_token = _mcp_gateway_server_name.set(scoped_server_name)
         try:
             yield
         finally:
-            _mcp_gateway_initialize_instructions.reset(tok)
+            _mcp_gateway_initialize_instructions.reset(instructions_token)
+            _mcp_gateway_server_name.reset(server_name_token)
 
     async def _get_tools_from_mcp_servers(  # noqa: PLR0915
         user_api_key_auth: Optional[UserAPIKeyAuth],
@@ -3620,6 +3637,7 @@ if MCP_AVAILABLE:
                 oauth2_headers,
                 raw_headers,
             ) = await extract_mcp_auth_context(scope, path)
+            scoped_server_endpoint = len(_get_mcp_servers_in_path(path) or []) == 1
 
             # Extract client IP for MCP access control
             _client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope))
@@ -3896,6 +3914,7 @@ if MCP_AVAILABLE:
                     user_api_key_auth,
                     mcp_servers,
                     _client_ip,
+                    scoped_server_endpoint=scoped_server_endpoint,
                 ):
                     await target_manager.handle_request(scope, receive, local_send)
                     if use_stateful and session_id and scope.get("method") == "DELETE":
@@ -3980,6 +3999,7 @@ if MCP_AVAILABLE:
                 oauth2_headers,
                 raw_headers,
             ) = await extract_mcp_auth_context(scope, path)
+            scoped_server_endpoint = len(_get_mcp_servers_in_path(path) or []) == 1
 
             # Extract client IP for MCP access control
             _sse_client_ip = IPAddressUtils.get_mcp_client_ip(StarletteRequest(scope))
@@ -4052,6 +4072,7 @@ if MCP_AVAILABLE:
                 user_api_key_auth,
                 mcp_servers,
                 _sse_client_ip,
+                scoped_server_endpoint=scoped_server_endpoint,
             ):
                 await sse_session_manager.handle_request(scope, receive, send)
         except MCPUpstreamAuthError as e:

litellm/proxy/_types.py

@@ -23,6 +23,7 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
 from litellm.types.integrations.slack_alerting import AlertType
 from litellm.types.llms.openai import (
     AllMessageValues,
+    ResponsesAPIResponse,
 )
 from litellm.types.mcp import (
     MCPAuthType,
@@ -2176,6 +2177,17 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
             "`statement_cache_size`). Keys here override any default LiteLLM sets."
         ),
     )
+    database_disable_prepared_statements: Optional[bool] = Field(
+        None,
+        description=(
+            "Disable server-side prepared statements by setting Prisma's "
+            "`pgbouncer=true` URL param. Use this for pgbouncer transaction-pooling "
+            "deployments, or to prevent the 'cached plan must not change result "
+            "type' error that pooled connections hit during rolling schema "
+            "migrations. An explicit `pgbouncer` in `database_extra_connection_params` "
+            "takes precedence."
+        ),
+    )
     database_type: Optional[Literal["dynamo_db"]] = Field(
         None, description="to use dynamodb instead of postgres db"
     )
@@ -3834,6 +3846,7 @@ PassThroughEndpointLoggingResultValues = Union[
     EmbeddingResponse,
     VideoObject,
     StandardPassThroughResponseObject,
+    ResponsesAPIResponse,
 ]
 
 

litellm/proxy/auth/auth_checks.py

@@ -3516,10 +3516,13 @@ async def _virtual_key_max_budget_check(
     if valid_token.max_budget is not None:
         from litellm.proxy.proxy_server import get_current_spend
 
+        fallback_spend = valid_token.spend or 0.0
+        counter_key = f"spend:key:{valid_token.token}"
+
         # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
         spend = await get_current_spend(
-            counter_key=f"spend:key:{valid_token.token}",
-            fallback_spend=valid_token.spend or 0.0,
+            counter_key=counter_key,
+            fallback_spend=fallback_spend,
         )
 
         ####################################

litellm/proxy/auth/auth_exception_handler.py

@@ -168,6 +168,16 @@ class UserAPIKeyAuthExceptionHandler:
                 )
             elif isinstance(e, ProxyException):
                 raise e
+            if PrismaDBExceptionHandler.is_database_service_unavailable_error(e):
+                raise ProxyException(
+                    message=(
+                        "Service Unavailable, the authentication database is "
+                        "temporarily unreachable. Please retry shortly."
+                    ),
+                    type=ProxyErrorTypes.no_db_connection,
+                    param="None",
+                    code=status.HTTP_503_SERVICE_UNAVAILABLE,
+                )
             raise ProxyException(
                 message="Authentication Error, " + str(e),
                 type=ProxyErrorTypes.auth_error,

litellm/proxy/auth/auth_utils.py

@@ -271,6 +271,11 @@ _BANNED_REQUEST_BODY_PARAMS: Tuple[str, ...] = (
     # tokens) to the attacker's host, or coerces the proxy into
     # authenticating against the attacker's host with admin secrets.
     "aws_bedrock_runtime_endpoint",
+    # Bedrock project/workspace association. Deployments pin this to
+    # enforce a data-retention policy, so a caller-supplied value would
+    # re-route the request's retention and accounting to any project
+    # reachable with the deployment's shared AWS credentials.
+    "aws_bedrock_project_id",
     # Provider-specific endpoint overrides that flow into the outbound
     # request via ``optional_params``. Same threat as ``api_base``:
     # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker

litellm/proxy/client/README.md

@@ -338,9 +338,9 @@ sequenceDiagram
 
 The CLI provides three authentication commands:
 
-- **`litellm-proxy login`** - Start SSO authentication flow
-- **`litellm-proxy logout`** - Clear stored authentication token
-- **`litellm-proxy whoami`** - Show current authentication status
+- **`lite login`** - Start SSO authentication flow
+- **`lite logout`** - Clear stored authentication token
+- **`lite whoami`** - Show current authentication status
 
 ### Authentication Flow Steps
 
@@ -382,14 +382,14 @@ Once authenticated, the CLI will automatically use the stored token for all requ
 
 ```bash
 # Login
-litellm-proxy login
+lite login
 
 # Use CLI without specifying API key
-litellm-proxy models list
+lite models list
 
 # Check authentication status
-litellm-proxy whoami
+lite whoami
 
 # Logout
-litellm-proxy logout
+lite logout
 ``` 

litellm/proxy/client/cli/README.md

@@ -22,11 +22,11 @@ The CLI can be configured using environment variables or command-line options:
 Example:
 
 ```bash
-litellm-proxy version
+lite version
 # or
-litellm-proxy --version
+lite --version
 # or
-litellm-proxy -v
+lite -v
 ```
 
 ## Commands
@@ -40,7 +40,7 @@ The CLI provides several commands for managing models on your LiteLLM proxy serv
 View all available models:
 
 ```bash
-litellm-proxy models list [--format table|json]
+lite models list [--format table|json]
 ```
 
 Options:
@@ -52,7 +52,7 @@ Options:
 Get detailed information about all models:
 
 ```bash
-litellm-proxy models info [options]
+lite models info [options]
 ```
 
 Options:
@@ -75,7 +75,7 @@ Default columns: `public_model`, `upstream_model`, `updated_at`
 Add a new model to the proxy:
 
 ```bash
-litellm-proxy models add <model-name> [options]
+lite models add <model-name> [options]
 ```
 
 Options:
@@ -86,7 +86,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.com -i description="GPT-4 model"
+lite models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.com -i description="GPT-4 model"
 ```
 
 #### Get Model Info
@@ -94,7 +94,7 @@ litellm-proxy models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.
 Get information about a specific model:
 
 ```bash
-litellm-proxy models get [--id MODEL_ID] [--name MODEL_NAME]
+lite models get [--id MODEL_ID] [--name MODEL_NAME]
 ```
 
 Options:
@@ -107,7 +107,7 @@ Options:
 Delete a model from the proxy:
 
 ```bash
-litellm-proxy models delete <model-id>
+lite models delete <model-id>
 ```
 
 #### Update Model
@@ -115,7 +115,7 @@ litellm-proxy models delete <model-id>
 Update an existing model's configuration:
 
 ```bash
-litellm-proxy models update <model-id> [options]
+lite models update <model-id> [options]
 ```
 
 Options:
@@ -128,7 +128,7 @@ Options:
 Import models from a YAML file:
 
 ```bash
-litellm-proxy models import models.yaml
+lite models import models.yaml
 ```
 
 Options:
@@ -142,31 +142,31 @@ Examples:
 1. Import all models from a YAML file:
 
 ```bash
-litellm-proxy models import models.yaml
+lite models import models.yaml
 ```
 
 2. Dry run (show what would be imported):
 
 ```bash
-litellm-proxy models import models.yaml --dry-run
+lite models import models.yaml --dry-run
 ```
 
 3. Only import models where the model name contains 'gpt':
 
 ```bash
-litellm-proxy models import models.yaml --only-models-matching-regex gpt
+lite models import models.yaml --only-models-matching-regex gpt
 ```
 
 4. Only import models with access group containing 'beta':
 
 ```bash
-litellm-proxy models import models.yaml --only-access-groups-matching-regex beta
+lite models import models.yaml --only-access-groups-matching-regex beta
 ```
 
 5. Combine both filters:
 
 ```bash
-litellm-proxy models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
+lite models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
 ```
 
 ### Credentials Management
@@ -178,7 +178,7 @@ The CLI provides commands for managing credentials on your LiteLLM proxy server:
 View all available credentials:
 
 ```bash
-litellm-proxy credentials list [--format table|json]
+lite credentials list [--format table|json]
 ```
 
 Options:
@@ -194,7 +194,7 @@ The table format displays:
 Create a new credential:
 
 ```bash
-litellm-proxy credentials create <credential-name> --info <json-string> --values <json-string>
+lite credentials create <credential-name> --info <json-string> --values <json-string>
 ```
 
 Options:
@@ -205,7 +205,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy credentials create azure-cred \
+lite credentials create azure-cred \
   --info '{"custom_llm_provider": "azure"}' \
   --values '{"api_key": "sk-123", "api_base": "https://example.azure.openai.com"}'
 ```
@@ -215,7 +215,7 @@ litellm-proxy credentials create azure-cred \
 Get information about a specific credential:
 
 ```bash
-litellm-proxy credentials get <credential-name>
+lite credentials get <credential-name>
 ```
 
 #### Delete Credential
@@ -223,7 +223,7 @@ litellm-proxy credentials get <credential-name>
 Delete a credential:
 
 ```bash
-litellm-proxy credentials delete <credential-name>
+lite credentials delete <credential-name>
 ```
 
 ### Keys Management
@@ -235,7 +235,7 @@ The CLI provides commands for managing API keys on your LiteLLM proxy server:
 View all API keys:
 
 ```bash
-litellm-proxy keys list [--format table|json] [options]
+lite keys list [--format table|json] [options]
 ```
 
 Options:
@@ -256,7 +256,7 @@ Options:
 Generate a new API key:
 
 ```bash
-litellm-proxy keys generate [options]
+lite keys generate [options]
 ```
 
 Options:
@@ -274,7 +274,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration 24h --key-alias my-key --team-id team123
+lite keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration 24h --key-alias my-key --team-id team123
 ```
 
 #### Delete Keys
@@ -282,7 +282,7 @@ litellm-proxy keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration
 Delete API keys by key or alias:
 
 ```bash
-litellm-proxy keys delete [--keys <comma-separated-keys>] [--key-aliases <comma-separated-aliases>]
+lite keys delete [--keys <comma-separated-keys>] [--key-aliases <comma-separated-aliases>]
 ```
 
 Options:
@@ -293,7 +293,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
+lite keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
 ```
 
 #### Get Key Info
@@ -301,7 +301,7 @@ litellm-proxy keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2
 Get information about a specific API key:
 
 ```bash
-litellm-proxy keys info --key <key-hash>
+lite keys info --key <key-hash>
 ```
 
 Options:
@@ -311,7 +311,7 @@ Options:
 Example:
 
 ```bash
-litellm-proxy keys info --key sk-key1
+lite keys info --key sk-key1
 ```
 
 ### User Management
@@ -323,7 +323,7 @@ The CLI provides commands for managing users on your LiteLLM proxy server:
 View all users:
 
 ```bash
-litellm-proxy users list
+lite users list
 ```
 
 #### Get User Info
@@ -331,7 +331,7 @@ litellm-proxy users list
 Get information about a specific user:
 
 ```bash
-litellm-proxy users get --id <user-id>
+lite users get --id <user-id>
 ```
 
 #### Create User
@@ -339,7 +339,7 @@ litellm-proxy users get --id <user-id>
 Create a new user:
 
 ```bash
-litellm-proxy users create --email user@example.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
+lite users create --email user@example.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
 ```
 
 #### Delete User
@@ -347,7 +347,7 @@ litellm-proxy users create --email user@example.com --role internal_user --alias
 Delete one or more users by user_id:
 
 ```bash
-litellm-proxy users delete <user-id-1> <user-id-2>
+lite users delete <user-id-1> <user-id-2>
 ```
 
 ### Chat Commands
@@ -359,7 +359,7 @@ The CLI provides commands for interacting with chat models through your LiteLLM
 Create a chat completion:
 
 ```bash
-litellm-proxy chat completions <model> [options]
+lite chat completions <model> [options]
 ```
 
 Arguments:
@@ -379,12 +379,12 @@ Examples:
 
 1. Simple completion:
 ```bash
-litellm-proxy chat completions gpt-4 -m "user:Hello, how are you?"
+lite chat completions gpt-4 -m "user:Hello, how are you?"
 ```
 
 2. Multi-message conversation:
 ```bash
-litellm-proxy chat completions gpt-4 \
+lite chat completions gpt-4 \
   -m "system:You are a helpful assistant" \
   -m "user:What's the capital of France?" \
   -m "assistant:The capital of France is Paris." \
@@ -393,7 +393,7 @@ litellm-proxy chat completions gpt-4 \
 
 3. With generation parameters:
 ```bash
-litellm-proxy chat completions gpt-4 \
+lite chat completions gpt-4 \
   -m "user:Write a story" \
   --temperature 0.7 \
   --max-tokens 500 \
@@ -409,7 +409,7 @@ The CLI provides commands for making direct HTTP requests to your LiteLLM proxy
 Make an HTTP request to any endpoint:
 
 ```bash
-litellm-proxy http request <method> <uri> [options]
+lite http request <method> <uri> [options]
 ```
 
 Arguments:
@@ -425,19 +425,46 @@ Examples:
 
 1. List models:
 ```bash
-litellm-proxy http request GET /models
+lite http request GET /models
 ```
 
 2. Create a chat completion:
 ```bash
-litellm-proxy http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
+lite http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}'
 ```
 
 3. Test connection with custom headers:
 ```bash
-litellm-proxy http request GET /health/test_connection -H "X-Custom-Header:value"
+lite http request GET /health/test_connection -H "X-Custom-Header:value"
 ```
 
+### Run a Coding Agent
+
+Launch a coding agent with all of its LLM traffic routed through your LiteLLM proxy. Each supported agent is its own command, so there is nothing to remember beyond the agent's name:
+
+```bash
+lite claude
+lite codex
+lite opencode
+```
+
+Anything you type after the agent name is forwarded to it untouched, so the usual flags keep working:
+
+```bash
+lite claude --resume
+lite codex exec "summarize the repo"
+```
+
+Each command resolves your LiteLLM key (logging in via SSO when none is stored and you are at a terminal; otherwise it expects `LITELLM_PROXY_API_KEY` or `--api-key`), checks the key against the proxy so bad credentials fail immediately instead of deep inside the agent, exports the environment variables the agent reads, then replaces itself with the agent process.
+
+The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol).
+
+Options (these belong to the wrapper, so put them before the agent's own flags):
+
+- `--skip-verify`: Skip the pre-launch key check (useful offline or with non-standard auth).
+
+To pin the model, pass the agent's own model flag (for example `lite claude --model my-proxy-model` or `lite codex -m my-proxy-model`), or export the variable the agent reads (`ANTHROPIC_MODEL` / `ANTHROPIC_SMALL_FAST_MODEL` for Claude Code); the wrapper preserves anything you already have set. Whatever model the agent ends up requesting must exist on the proxy, since requests land on the proxy's `/v1/messages` (Anthropic) or `/v1/chat/completions` and `/v1/responses` (OpenAI) endpoints.
+
 ## Environment Variables
 
 The CLI respects the following environment variables:
@@ -450,37 +477,37 @@ The CLI respects the following environment variables:
 1. List all models in table format:
 
 ```bash
-litellm-proxy models list
+lite models list
 ```
 
 2. Add a new model with parameters:
 
 ```bash
-litellm-proxy models add gpt-4 -p api_key=sk-123 -p max_tokens=2048
+lite models add gpt-4 -p api_key=sk-123 -p max_tokens=2048
 ```
 
 3. Get model information in JSON format:
 
 ```bash
-litellm-proxy models info --format json
+lite models info --format json
 ```
 
 4. Update model parameters:
 
 ```bash
-litellm-proxy models update model-123 -p temperature=0.7 -i description="Updated model"
+lite models update model-123 -p temperature=0.7 -i description="Updated model"
 ```
 
 5. List all credentials in table format:
 
 ```bash
-litellm-proxy credentials list
+lite credentials list
 ```
 
 6. Create a new credential for Azure:
 
 ```bash
-litellm-proxy credentials create azure-prod \
+lite credentials create azure-prod \
   --info '{"custom_llm_provider": "azure"}' \
   --values '{"api_key": "sk-123", "api_base": "https://prod.azure.openai.com"}'
 ```
@@ -488,7 +515,7 @@ litellm-proxy credentials create azure-prod \
 7. Make a custom HTTP request:
 
 ```bash
-litellm-proxy http request POST /chat/completions \
+lite http request POST /chat/completions \
   -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' \
   -H "X-Custom-Header:value"
 ```
@@ -497,29 +524,29 @@ litellm-proxy http request POST /chat/completions \
 
 ```bash
 # List users
-litellm-proxy users list
+lite users list
 
 # Get user info
-litellm-proxy users get --id u1
+lite users get --id u1
 
 # Create a user
-litellm-proxy users create --email a@b.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
+lite users create --email a@b.com --role internal_user --alias "Alice" --team team1 --max-budget 100.0
 
 # Delete users
-litellm-proxy users delete u1 u2
+lite users delete u1 u2
 ```
 
 9. Import models from a YAML file (with filters):
 
 ```bash
 # Only import models where the model name contains 'gpt'
-litellm-proxy models import models.yaml --only-models-matching-regex gpt
+lite models import models.yaml --only-models-matching-regex gpt
 
 # Only import models with access group containing 'beta'
-litellm-proxy models import models.yaml --only-access-groups-matching-regex beta
+lite models import models.yaml --only-access-groups-matching-regex beta
 
 # Combine both filters
-litellm-proxy models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
+lite models import models.yaml --only-models-matching-regex gpt --only-access-groups-matching-regex beta
 ```
 
 ## Error Handling

litellm/proxy/client/cli/commands/agents.py

@@ -0,0 +1,303 @@
+import os
+import shutil
+import sys
+from typing import Callable, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple
+
+import click
+import requests
+
+from .auth import get_stored_api_key, login
+
+ANTHROPIC_BASE_URL_ENV = "ANTHROPIC_BASE_URL"
+ANTHROPIC_AUTH_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN"
+ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY"
+OPENAI_BASE_URL_ENV = "OPENAI_BASE_URL"
+OPENAI_API_KEY_ENV = "OPENAI_API_KEY"
+
+PROFILE_ANTHROPIC = "anthropic"
+PROFILE_OPENAI = "openai"
+
+_KNOWN_AGENTS: Dict[str, Tuple[str, FrozenSet[str]]] = {
+    "claude": ("Claude Code", frozenset({PROFILE_ANTHROPIC})),
+    "codex": ("Codex", frozenset({PROFILE_OPENAI})),
+    "opencode": ("OpenCode", frozenset({PROFILE_OPENAI})),
+}
+
+_INSTALL_DOCS: Dict[str, str] = {
+    "claude": "https://docs.claude.com/en/docs/claude-code/setup",
+    "codex": "https://developers.openai.com/codex/cli",
+    "opencode": "https://opencode.ai/docs",
+}
+
+CODEX_PROXY_PROVIDER = "litellm"
+
+
+class AgentRunError(Exception):
+    """Raised for any user-actionable failure while preparing to run an agent."""
+
+
+def agent_profile(command: str) -> Tuple[str, FrozenSet[str]]:
+    """Return the (display name, env profiles) for a wrapped command.
+
+    Known agents map to the API family they speak. Anything else gets both
+    families so it works regardless of which env vars the tool reads.
+    """
+    base = os.path.basename(command)
+    if base in _KNOWN_AGENTS:
+        return _KNOWN_AGENTS[base]
+    return base, frozenset({PROFILE_ANTHROPIC, PROFILE_OPENAI})
+
+
+def build_agent_env(
+    base_env: Mapping[str, str],
+    base_url: str,
+    api_key: str,
+    profiles: FrozenSet[str],
+) -> Dict[str, str]:
+    """Return a copy of base_env wired to route the agent through the proxy.
+
+    Anthropic clients (Claude Code) append /v1/messages to ANTHROPIC_BASE_URL,
+    so it stays the bare proxy root; OpenAI clients (Codex, OpenCode) expect the
+    /v1 suffix on OPENAI_BASE_URL. ANTHROPIC_API_KEY is dropped so a stray
+    Anthropic key cannot win over the bearer token we set.
+    """
+    env = dict(base_env)
+    root = base_url.rstrip("/")
+    if PROFILE_ANTHROPIC in profiles:
+        env[ANTHROPIC_BASE_URL_ENV] = root
+        env[ANTHROPIC_AUTH_TOKEN_ENV] = api_key
+        env.pop(ANTHROPIC_API_KEY_ENV, None)
+    if PROFILE_OPENAI in profiles:
+        env[OPENAI_BASE_URL_ENV] = root + "/v1"
+        env[OPENAI_API_KEY_ENV] = api_key
+    return env
+
+
+def _codex_proxy_args(base_url: str) -> List[str]:
+    """Codex `-c` overrides that point it at the proxy.
+
+    Codex ignores OPENAI_BASE_URL (it always dials api.openai.com), so the env
+    profile alone cannot route it. It does honor a custom provider, so define one
+    inline; supports_websockets=false forces the HTTP/SSE Responses transport
+    because the proxy does not speak the Responses WebSocket protocol. The key is
+    read from OPENAI_API_KEY, which build_agent_env already exports.
+    """
+    root = base_url.rstrip("/") + "/v1"
+    provider = f"model_providers.{CODEX_PROXY_PROVIDER}"
+    return [
+        "-c",
+        f'model_provider="{CODEX_PROXY_PROVIDER}"',
+        "-c",
+        f'{provider}.name="LiteLLM proxy"',
+        "-c",
+        f'{provider}.base_url="{root}"',
+        "-c",
+        f'{provider}.env_key="{OPENAI_API_KEY_ENV}"',
+        "-c",
+        f'{provider}.wire_api="responses"',
+        "-c",
+        f"{provider}.supports_websockets=false",
+    ]
+
+
+_PROXY_ARGS: Dict[str, Callable[[str], List[str]]] = {
+    "codex": _codex_proxy_args,
+}
+
+
+def agent_launch_args(command: str, base_url: str) -> List[str]:
+    """Extra CLI args an agent needs to actually honor the proxy.
+
+    Claude Code and OpenCode respect the exported env vars, so they get nothing
+    here; Codex needs its provider pointed via config overrides.
+    """
+    builder = _PROXY_ARGS.get(os.path.basename(command))
+    return builder(base_url) if builder else []
+
+
+def verify_proxy_key(
+    base_url: str,
+    api_key: str,
+    *,
+    get: Callable[..., requests.Response] = requests.get,
+) -> None:
+    """Probe the proxy with the key so bad creds fail here, not inside the agent.
+
+    Raises AgentRunError when the proxy is unreachable or rejects the key. Other
+    non-2xx responses are tolerated; the agent's own call is the real test.
+    """
+    url = base_url.rstrip("/") + "/v1/models"
+    try:
+        resp = get(url, headers={"Authorization": f"Bearer {api_key}"}, timeout=10)
+    except requests.RequestException as e:
+        raise AgentRunError(
+            f"Could not reach the LiteLLM proxy at {base_url.rstrip('/')}: {e}. "
+            "Is it running, and is --base-url (or LITELLM_PROXY_URL) correct?"
+        )
+    if resp.status_code in (401, 403):
+        raise AgentRunError(
+            f"LiteLLM rejected your key (HTTP {resp.status_code}). "
+            "Run `lite login` to refresh it, or pass a valid --api-key."
+        )
+
+
+def _exec(path: str, args: Sequence[str], env: Mapping[str, str]) -> None:
+    os.execvpe(path, list(args), dict(env))
+
+
+def _restore_controlling_terminal() -> None:
+    """Reattach the controlling terminal to stdin before handing off to the agent.
+
+    Completing the browser SSO login can leave stdin detached from the terminal,
+    which makes a TUI agent like Claude Code start in non-interactive mode and
+    exit immediately. Reopening /dev/tty onto fd 0 gives the agent a live
+    terminal; when stdin is still a tty (no login happened) this is a no-op.
+    """
+    if sys.stdin.isatty():
+        return
+    try:
+        fd = os.open("/dev/tty", os.O_RDONLY)
+    except OSError:
+        return
+    try:
+        os.dup2(fd, 0)
+    finally:
+        os.close(fd)
+
+
+def run_agent(
+    base_url: str,
+    api_key: str,
+    command: Sequence[str],
+    *,
+    skip_verify: bool = False,
+    base_env: Optional[Mapping[str, str]] = None,
+    which: Callable[[str], Optional[str]] = shutil.which,
+    verify: Callable[[str, str], None] = verify_proxy_key,
+    launcher: Callable[[str, Sequence[str], Mapping[str, str]], None] = _exec,
+    reattach_terminal: Optional[Callable[[], None]] = None,
+) -> None:
+    """Validate, wire the environment, and hand off to the agent.
+
+    On success this replaces the current process and never returns. Raises
+    AgentRunError for missing binaries, an unreachable proxy, or a rejected key.
+    reattach_terminal, when given, runs just before handoff to restore stdin.
+    """
+    if not command:
+        raise AgentRunError("Nothing to run.")
+
+    _, profiles = agent_profile(command[0])
+    binary = which(command[0])
+    if binary is None:
+        docs = _INSTALL_DOCS.get(os.path.basename(command[0]))
+        hint = f" Install it first: {docs}" if docs else ""
+        raise AgentRunError(f"Could not find `{command[0]}` on your PATH.{hint}")
+
+    if not skip_verify:
+        verify(base_url, api_key)
+
+    env = build_agent_env(
+        base_env if base_env is not None else os.environ,
+        base_url,
+        api_key,
+        profiles,
+    )
+    extra_args = agent_launch_args(command[0], base_url)
+    if reattach_terminal is not None:
+        reattach_terminal()
+    launcher(binary, [command[0], *extra_args, *command[1:]], env)
+
+
+def _is_interactive() -> bool:
+    return sys.stdin.isatty()
+
+
+def _resolve_api_key(ctx: click.Context) -> str:
+    base_url = ctx.obj["base_url"]
+    api_key = ctx.obj.get("api_key")
+    if api_key:
+        return api_key
+
+    if not _is_interactive():
+        raise click.ClickException(
+            "No LiteLLM key found. Set LITELLM_PROXY_API_KEY (or pass --api-key) for "
+            "non-interactive use, or run `lite login` from a terminal."
+        )
+
+    click.echo("No LiteLLM credentials found; starting login...")
+    ctx.invoke(login)
+    api_key = get_stored_api_key(expected_base_url=base_url)
+    if not api_key:
+        raise click.ClickException(
+            "Login did not produce an API key; cannot start the agent."
+        )
+    return api_key
+
+
+_SKIP_VERIFY_HELP = "Skip the pre-launch key check against the proxy."
+
+
+def _launch(
+    ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool
+) -> None:
+    base_url = ctx.obj["base_url"]
+    started_interactive = _is_interactive()
+    api_key = _resolve_api_key(ctx)
+
+    display_name, _ = agent_profile(binary)
+    click.echo(
+        f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}"
+    )
+
+    try:
+        run_agent(
+            base_url,
+            api_key,
+            [binary, *args],
+            skip_verify=skip_verify,
+            reattach_terminal=(
+                _restore_controlling_terminal if started_interactive else None
+            ),
+        )
+    except AgentRunError as e:
+        raise click.ClickException(str(e))
+
+
+def _make_agent_command(binary: str, display_name: str) -> click.Command:
+    @click.command(
+        name=binary,
+        context_settings={"ignore_unknown_options": True},
+        short_help=f"Run {display_name} through your LiteLLM proxy",
+    )
+    @click.option("--skip-verify", is_flag=True, default=False, help=_SKIP_VERIFY_HELP)
+    @click.argument("args", nargs=-1, type=click.UNPROCESSED)
+    @click.pass_context
+    def _command(ctx: click.Context, skip_verify: bool, args: Sequence[str]) -> None:
+        _launch(ctx, binary, list(args), skip_verify=skip_verify)
+
+    _command.help = (
+        f"Run {display_name} routed through your LiteLLM proxy.\n\n"
+        f"Logs in with LiteLLM if needed, verifies your key against the proxy, "
+        f"exports the env vars {binary} reads, then hands off. Any arguments are "
+        f"forwarded to `{binary}`."
+    )
+    return _command
+
+
+def agent_commands() -> List[click.Command]:
+    """Build one top-level command per known agent, e.g. `lite claude`."""
+    return [
+        _make_agent_command(binary, name)
+        for binary, (name, _profiles) in _KNOWN_AGENTS.items()
+    ]
+
+
+__all__ = [
+    "agent_commands",
+    "run_agent",
+    "build_agent_env",
+    "agent_launch_args",
+    "verify_proxy_key",
+    "agent_profile",
+    "AgentRunError",
+]

litellm/proxy/client/cli/commands/auth.py

@@ -624,7 +624,7 @@ def whoami():
     token_data = load_token()
 
     if not token_data:
-        click.echo("❌ Not authenticated. Run 'litellm-proxy login' to authenticate.")
+        click.echo("❌ Not authenticated. Run 'lite login' to authenticate.")
         return
 
     click.echo("✅ Authenticated")

litellm/proxy/client/cli/commands/chat.py

@@ -122,13 +122,13 @@ def chat(
     Examples:
 
         # Chat with a specific model
-        litellm-proxy chat gpt-4
+        lite chat gpt-4
 
         # Chat without specifying model (will show model selection)
-        litellm-proxy chat
+        lite chat
 
         # Chat with custom settings
-        litellm-proxy chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
+        lite chat gpt-4 --temperature 0.9 --system "You are a helpful coding assistant"
     """
     console = Console()
 

litellm/proxy/client/cli/interface.py

@@ -80,6 +80,8 @@ def styled_prompt():
 
 def show_commands():
     """Display available commands."""
+    from .commands.agents import agent_commands
+
     commands = [
         ("login", "Authenticate with the LiteLLM proxy server"),
         ("logout", "Clear stored authentication"),
@@ -91,6 +93,9 @@ def show_commands():
         ("keys", "Manage API keys"),
         ("teams", "Manage teams and team assignments"),
         ("users", "Manage users"),
+    ]
+    commands += [(c.name, c.get_short_help_str()) for c in agent_commands()]
+    commands += [
         ("version", "Show version information"),
         ("help", "Show this help message"),
         ("quit", "Exit the interactive session"),
@@ -156,7 +161,7 @@ def execute_command(user_input: str, ctx: click.Context):
     # Execute the command
     try:
         # Create a new argument list for click to parse
-        sys.argv = ["litellm-proxy"] + [command] + args
+        sys.argv = ["lite"] + [command] + args
 
         # Get the command object and invoke it
         cmd = cli.commands[command]

litellm/proxy/client/cli/main.py

@@ -7,6 +7,7 @@ import click
 from litellm._version import version as litellm_version
 from litellm.proxy.client.health import HealthManagementClient
 
+from .commands.agents import agent_commands
 from .commands.auth import get_stored_api_key, login, logout, whoami
 from .commands.chat import chat
 from .commands.credentials import credentials
@@ -112,6 +113,9 @@ cli.add_command(keys)
 cli.add_command(teams)
 # Add the users command group
 cli.add_command(users)
+# Add a top-level command per coding agent (claude, codex, opencode, ...)
+for agent_command in agent_commands():
+    cli.add_command(agent_command)
 
 
 if __name__ == "__main__":

litellm/proxy/common_request_processing.py

@@ -1949,12 +1949,26 @@ class ProxyBaseLLMRequestProcessing:
                 code=status.HTTP_400_BAD_REQUEST,
                 headers=headers,
             )
+        # Extract status_code from the exception if it carries one.
+        # Provider exceptions (NotFoundError, BadRequestError, GeminiError,
+        # VertexAIError, etc.) all have a status_code attribute reflecting
+        # the upstream API response. Use it to return the correct HTTP code
+        # instead of defaulting to 500.
+        _exc_status_code = getattr(e, "status_code", None)
+        if (
+            _exc_status_code is not None
+            and isinstance(_exc_status_code, int)
+            and 400 <= _exc_status_code <= 599
+        ):
+            _code = _exc_status_code
+        else:
+            _code = status.HTTP_500_INTERNAL_SERVER_ERROR
         raise ProxyException(
             message=getattr(e, "message", error_msg),
             type=getattr(e, "type", "None"),
             param=getattr(e, "param", "None"),
             openai_code=getattr(e, "code", None),
-            code=getattr(e, "status_code", 500),
+            code=_code,
             provider_specific_fields=getattr(e, "provider_specific_fields", None),
             headers=headers,
         )

litellm/proxy/common_utils/callback_utils.py

@@ -40,8 +40,10 @@ def initialize_callbacks_on_proxy(  # noqa: PLR0915
     premium_user: bool,
     config_file_path: str,
     litellm_settings: dict,
-    callback_specific_params: dict = {},
+    callback_specific_params: Optional[dict] = None,
 ):
+    if not isinstance(callback_specific_params, dict):
+        callback_specific_params = {}
     from litellm.integrations.custom_logger import CustomLogger
     from litellm.litellm_core_utils.logging_callback_manager import (
         LoggingCallbackManager,
@@ -166,7 +168,12 @@ def initialize_callbacks_on_proxy(  # noqa: PLR0915
                 )
 
                 init_params = {}
-                if "lakera_prompt_injection" in callback_specific_params:
+                if (
+                    "lakera_prompt_injection" in callback_specific_params
+                    and isinstance(
+                        callback_specific_params["lakera_prompt_injection"], dict
+                    )
+                ):
                     init_params = callback_specific_params["lakera_prompt_injection"]
                 lakera_moderations_object = lakeraAI_Moderation(**init_params)
                 imported_list.append(lakera_moderations_object)

litellm/proxy/common_utils/html_forms/cli_sso_success.py

@@ -135,7 +135,7 @@ def render_cli_sso_success_page() -> str:
                 font-size: 14px;
             }}
 
-            .countdown {{
+            .status {{
                 color: #64748b;
                 font-size: 14px;
                 font-weight: 500;
@@ -183,23 +183,11 @@ def render_cli_sso_success_page() -> str:
                 <p>You can now use LiteLLM CLI commands with your authenticated session.</p>
             </div>
             
-            <div class="countdown" id="countdown">This window will close in 3 seconds...</div>
+            <div class="status">You can now close this window and return to your terminal.</div>
         </div>
-        
+
         <script>
-            let seconds = 3;
-            const countdownElement = document.getElementById('countdown');
-            
-            const countdown = setInterval(function() {{
-                seconds--;
-                if (seconds > 0) {{
-                    countdownElement.textContent = `This window will close in ${{seconds}} second${{seconds === 1 ? '' : 's'}}...`;
-                }} else {{
-                    countdownElement.textContent = 'Closing...';
-                    clearInterval(countdown);
-                    window.close();
-                }}
-            }}, 1000);
+            window.close();
         </script>
     </body>
     </html>

litellm/proxy/db/exception_handler.py

@@ -109,6 +109,92 @@ class PrismaDBExceptionHandler:
             return True
         return False
 
+    @staticmethod
+    def is_prisma_engine_internal_error(e: Exception) -> bool:
+        """True iff ``e`` is a non-``PrismaError`` exception raised from inside
+        prisma-client-py's query-engine layer.
+
+        During the instant a DB connection is torn down, the query engine can
+        return a malformed error payload (``user_facing_error.meta`` is
+        ``null``). prisma-client-py's ``handle_response_errors`` then crashes
+        with ``AttributeError: 'NoneType' object has no attribute 'get'``
+        before it can raise the proper P1001 "can't reach database server"
+        error. That AttributeError carries no connection keyword, so it can't
+        be matched by message; identify it by its ``prisma.engine`` origin
+        instead.
+
+        Recognized ``PrismaError`` subclasses are excluded: connectivity ones
+        are already classified by type/keyword above, and data-layer ones
+        (the DB IS reachable) must stay 401.
+        """
+        import prisma
+
+        if isinstance(e, prisma.errors.PrismaError):
+            return False
+        tb = getattr(e, "__traceback__", None)
+        while tb is not None:
+            if tb.tb_frame.f_globals.get("__name__", "").startswith("prisma.engine"):
+                return True
+            tb = tb.tb_next
+        return False
+
+    @staticmethod
+    def is_database_service_unavailable_error(e: Exception) -> bool:
+        """True iff the exception means the database could not answer at the
+        infrastructure level (connection refused, socket/interface failure,
+        timeout) rather than a genuine auth failure (key not found) or a
+        data-layer error (the DB IS reachable and rejected the data).
+
+        Auth must answer 401 only for a key the DB confirms is invalid. When
+        the DB itself is unreachable, the request has to surface as 503 so
+        callers retry instead of treating valid keys as invalid during an
+        outage.
+
+        Note: prisma-client-py mislabels the P1001 "can't reach database
+        server" connectivity failure as a ``DataError`` (a data-layer type),
+        so a type-only check misses real outages. ``is_database_transport_error``
+        keyword-matches the connection message and catches that masquerade,
+        while genuine data errors (no connection keyword) correctly stay 401.
+
+        The Postgres "cached plan must not change result type" error is matched
+        here, not in ``is_database_transport_error``: it is a transient stale-DB-
+        state condition (not an invalid key), but the connection is healthy so it
+        must not trigger a reconnect.
+
+        A non-``PrismaError`` raised from inside the prisma query engine (e.g.
+        the ``AttributeError`` from ``handle_response_errors`` when the engine
+        returns a malformed error payload mid-tear-down) is also treated as
+        unavailable; see ``is_prisma_engine_internal_error``.
+        """
+        import asyncio
+
+        if PrismaDBExceptionHandler.is_database_connection_error(e):
+            return True
+        if PrismaDBExceptionHandler.is_database_transport_error(e):
+            return True
+        if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e):
+            return True
+        if "cached plan must not change result type" in str(e).lower():
+            return True
+
+        # OSError already covers ConnectionError and (Py3.3+) TimeoutError.
+        # asyncio.TimeoutError is a distinct class before Py3.11.
+        if isinstance(e, (OSError, asyncio.TimeoutError)):
+            return True
+
+        try:
+            import asyncpg
+        except ImportError:
+            return False
+
+        return isinstance(
+            e,
+            (
+                asyncpg.exceptions.PostgresConnectionError,
+                asyncpg.exceptions.InterfaceError,
+            ),
+        )
+
     @staticmethod
     def handle_db_exception(e: Exception):
         """

litellm/proxy/db/tool_registry_writer.py

@@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ToolDiscoveryQueueItem
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.repositories.table_repositories import ToolRepository
 from litellm.types.tool_management import (
@@ -309,7 +310,11 @@ class ToolPolicyRegistry:
     async def sync_tool_policy_from_db(self, prisma_client: "PrismaClient") -> None:
         """Load all tool policies and object-permission blocked_tools from DB."""
         try:
-            tools = await ToolRepository(prisma_client).table.find_many()
+            tools = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ToolRepository(prisma_client).table.find_many(),
+                reason="sync_tool_policy_from_db_tools_lookup_failure",
+            )
             self._tool_input_policies = {
                 row.tool_name: getattr(row, "input_policy", "untrusted") or "untrusted"
                 for row in tools
@@ -319,7 +324,11 @@ class ToolPolicyRegistry:
                 for row in tools
             }
 
-            perms = await ObjectPermissionRepository(prisma_client).table.find_many()
+            perms = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ObjectPermissionRepository(prisma_client).table.find_many(),
+                reason="sync_tool_policy_from_db_perms_lookup_failure",
+            )
             self._blocked_tools_by_op_id = {}
             for row in perms:
                 op_id = getattr(row, "object_permission_id", None)

litellm/proxy/guardrails/guardrail_hooks/presidio.py

@@ -1225,6 +1225,39 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
             for chunk in all_chunks:
                 yield chunk
 
+    @staticmethod
+    def _unmask_sse_bytes_chunk(chunk: bytes, pii_tokens: Dict[str, str]) -> bytes:
+        try:
+            text = chunk.decode("utf-8")
+        except UnicodeDecodeError:
+            return chunk
+
+        result_lines: List[str] = []
+        for line in text.split("\n"):
+            line = line.rstrip("\r")
+            if line.startswith("data: ") and line != "data: [DONE]":
+                raw_json = line[6:]
+                try:
+                    event = json.loads(raw_json)
+                    delta = event.get("delta") if isinstance(event, dict) else None
+                    if (
+                        isinstance(delta, dict)
+                        and event.get("type") == "content_block_delta"
+                        and delta.get("type") == "text_delta"
+                        and isinstance(delta.get("text"), str)
+                    ):
+                        unmasked = _OPTIONAL_PresidioPIIMasking._unmask_pii_text(
+                            delta["text"], pii_tokens
+                        )
+                        if unmasked != delta["text"]:
+                            event["delta"]["text"] = unmasked
+                            line = "data: " + json.dumps(event, ensure_ascii=False)
+                except (json.JSONDecodeError, KeyError, TypeError):
+                    pass
+            result_lines.append(line)
+
+        return "\n".join(result_lines).encode("utf-8")
+
     async def _stream_pii_unmasking(
         self,
         response: Any,
@@ -1237,13 +1270,19 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
         from litellm.main import stream_chunk_builder
         from litellm.types.utils import ModelResponse
 
+        metadata = (request_data.get("metadata") or {}) if request_data else {}
+        pii_tokens: Dict[str, str] = metadata.get("pii_tokens", {})
+
         remaining_chunks: List[ModelResponseStream] = []
         try:
             async for chunk in response:
                 if isinstance(chunk, ModelResponseStream):
                     remaining_chunks.append(chunk)
                 elif isinstance(chunk, bytes):
-                    yield chunk  # type: ignore[misc]
+                    if pii_tokens:
+                        yield self._unmask_sse_bytes_chunk(chunk, pii_tokens)  # type: ignore[misc]
+                    else:
+                        yield chunk  # type: ignore[misc]
                     continue
 
             if not remaining_chunks:

litellm/proxy/hooks/batch_rate_limiter.py

@@ -531,11 +531,17 @@ class _PROXY_BatchRateLimiter(CustomLogger):
             # Check if this is a managed file (base64 encoded unified file ID)
             from litellm.proxy.openai_files_endpoints.common_utils import (
                 _is_base64_encoded_unified_file_id,
+                get_models_from_unified_file_id,
             )
 
             # Managed files require bypassing the HTTP endpoint (which runs access-check hooks)
             # and calling the managed files hook directly with the user's credentials.
             is_managed_file = _is_base64_encoded_unified_file_id(file_id)
+            target_model_names = (
+                get_models_from_unified_file_id(is_managed_file)
+                if is_managed_file
+                else []
+            )
             if is_managed_file and user_api_key_dict is not None:
                 file_content = await self._fetch_managed_file_content(
                     file_id=file_id,
@@ -573,6 +579,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
                 await self._enforce_batch_file_model_access(
                     user_api_key_dict=user_api_key_dict,
                     file_content_as_dict=file_content_as_dict,
+                    target_model_names=target_model_names or None,
                 )
 
             input_file_usage = _get_batch_job_input_file_usage(
@@ -608,9 +615,13 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         self,
         user_api_key_dict: UserAPIKeyAuth,
         file_content_as_dict: List[dict],
+        target_model_names: Optional[List[str]] = None,
     ) -> None:
-        """Reject the batch if the caller is not authorized for every
-        ``body.model`` named inside the JSONL.
+        """Reject the batch if the caller is not authorized for the upload target.
+
+        For managed files, ``target_model_names`` (from the unified file id) is
+        the proxy alias the file was uploaded for and is used directly for auth.
+        For legacy/non-managed files, falls back to ``body.model`` values in the JSONL.
 
         Reuses standard auth helpers so the same model access rules the proxy
         enforces on `/chat/completions` apply here.
@@ -627,9 +638,12 @@ class _PROXY_BatchRateLimiter(CustomLogger):
         from litellm.proxy.proxy_server import proxy_logging_obj
         from litellm.proxy.proxy_server import user_api_key_cache
 
-        models = _get_models_from_batch_input_file_content(file_content_as_dict)
-        if not models:
-            return
+        if target_model_names:
+            models = target_model_names
+        else:
+            models = _get_models_from_batch_input_file_content(file_content_as_dict)
+            if not models:
+                return
 
         team_object = None
         if (
@@ -660,12 +674,7 @@ class _PROXY_BatchRateLimiter(CustomLogger):
 
         llm_model_list = llm_router.model_list if llm_router is not None else None
         for model in models:
-            # body.model may be the provider id after replace_model_in_jsonl; map to proxy model_name for auth.
             model_to_check = model
-            if llm_router is not None:
-                proxy_model_name = llm_router.resolve_model_name_from_model_id(model)
-                if proxy_model_name is not None:
-                    model_to_check = proxy_model_name
             try:
                 if team_object is not None:
                     try:

litellm/proxy/hooks/litellm_skills/main.py

@@ -19,7 +19,7 @@ Usage:
     response = await litellm.acompletion(
         model="gpt-4o-mini",
         messages=[{"role": "user", "content": "Create a bouncing ball GIF"}],
-        container={"skills": [{"skill_id": "litellm:skill_abc123"}]},
+        container={"skills": [{"skill_id": "litellm_skill_abc123"}]},
     )
     # Response includes file_ids for generated files
 """
@@ -31,6 +31,7 @@ from typing import Any, Dict, List, Optional, Union
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.llms.litellm_proxy.skills.constants import LITELLM_SKILL_ID_PREFIX
 from litellm.llms.litellm_proxy.skills.prompt_injection import (
     SkillPromptInjectionHandler,
 )
@@ -43,7 +44,7 @@ class SkillsInjectionHook(CustomLogger):
     Pre/Post-call hook that processes skills from container.skills parameter.
 
     Pre-call (async_pre_call_hook):
-    - Skills with 'litellm:' prefix are fetched from LiteLLM DB
+    - Skills with 'litellm_skill_' prefix are fetched from LiteLLM DB
     - For Anthropic models: native skills pass through, LiteLLM skills converted to tools
     - For non-Anthropic models: LiteLLM skills are converted to tools + execute_code tool
 
@@ -78,7 +79,7 @@ class SkillsInjectionHook(CustomLogger):
         Process skills from container.skills before the LLM call.
 
         1. Check if container.skills exists in request
-        2. Separate skills by prefix (litellm: vs native)
+        2. Separate skills by prefix (litellm_skill_ vs native)
         3. Fetch LiteLLM skills from database
         4. For Anthropic: keep native skills in container
         5. For non-Anthropic: convert LiteLLM skills to tools, inject content, add execute_code
@@ -108,7 +109,7 @@ class SkillsInjectionHook(CustomLogger):
                 continue
 
             skill_id = skill.get("skill_id", "")
-            if skill_id.startswith("litellm_"):
+            if skill_id.startswith(LITELLM_SKILL_ID_PREFIX):
                 # Fetch from LiteLLM DB
                 db_skill = await self._fetch_skill_from_db(
                     skill_id,
@@ -287,7 +288,7 @@ class SkillsInjectionHook(CustomLogger):
         Fetch a skill from the LiteLLM database.
 
         Args:
-            skill_id: The skill ID (without 'litellm:' prefix)
+            skill_id: The skill ID (including the 'litellm_skill_' prefix)
 
         Returns:
             LiteLLM_SkillsTable or None if not found
@@ -382,10 +383,10 @@ class SkillsInjectionHook(CustomLogger):
         has_executable_tool = False
         for tc in tool_calls:
             tool_name = tc.get("name", "")
-            # Execute if it's litellm_code_execution OR a skill tool (skill_xxx)
+            # Execute if it's litellm_code_execution OR a skill tool (litellm_skill_xxx)
             if (
                 tool_name == LiteLLMInternalTools.CODE_EXECUTION.value
-                or tool_name.startswith("skill_")
+                or tool_name.startswith(LITELLM_SKILL_ID_PREFIX)
             ):
                 has_executable_tool = True
                 break
@@ -543,7 +544,7 @@ class SkillsInjectionHook(CustomLogger):
                     result = await self._execute_code(
                         code, skill_files, executor, generated_files
                     )
-                elif tool_name.startswith("skill_"):
+                elif tool_name.startswith(LITELLM_SKILL_ID_PREFIX):
                     # Skill tool - execute the skill's code
                     result = await self._execute_skill_tool(
                         tool_name, tool_input, skill_files, executor, generated_files

litellm/proxy/hooks/parallel_request_limiter.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel
 from typing_extensions import TypedDict
 
 import litellm
-from litellm import DualCache, ModelResponse
+from litellm import DualCache, EmbeddingResponse, ModelResponse, TextCompletionResponse
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
@@ -570,7 +570,9 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
 
             total_tokens = 0
 
-            if isinstance(response_obj, ModelResponse):
+            if isinstance(
+                response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)
+            ):
                 total_tokens = response_obj.usage.total_tokens  # type: ignore
 
             # ------------
@@ -659,7 +661,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_user_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (
@@ -692,7 +697,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_team_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (
@@ -725,7 +733,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
             if user_api_key_end_user_id is not None:
                 total_tokens = 0
 
-                if isinstance(response_obj, ModelResponse):
+                if isinstance(
+                    response_obj,
+                    (ModelResponse, EmbeddingResponse, TextCompletionResponse),
+                ):
                     total_tokens = response_obj.usage.total_tokens  # type: ignore
 
                 request_count_api_key = (

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -39,7 +39,13 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import (
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
 from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject
-from litellm.types.utils import CallTypes, ModelResponse, Usage
+from litellm.types.utils import (
+    CallTypes,
+    EmbeddingResponse,
+    ModelResponse,
+    TextCompletionResponse,
+    Usage,
+)
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -2736,9 +2742,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
 
         # Get total tokens from response
         total_tokens = 0
-        # spot fix for /responses api
-        if isinstance(response_obj, ModelResponse) or isinstance(
-            response_obj, BaseLiteLLMOpenAIResponseObject
+        if isinstance(
+            response_obj,
+            (
+                ModelResponse,
+                EmbeddingResponse,
+                TextCompletionResponse,
+                BaseLiteLLMOpenAIResponseObject,
+            ),
         ):
             _usage = getattr(response_obj, "usage", None)
             total_tokens = self._get_total_tokens_from_usage(

litellm/proxy/image_endpoints/endpoints.py

@@ -173,6 +173,16 @@ async def image_generation(
             )
         )
 
+        # Call response headers hook (matches base_process_llm_request behavior)
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            fastapi_response.headers.update(callback_headers)
+
         return response
     except Exception as e:
         await proxy_logging_obj.post_call_failure_hook(

litellm/proxy/management_endpoints/cache_settings_endpoints.py

@@ -27,6 +27,7 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.repositories.table_repositories import CacheConfigRepository
 from litellm.types.management_endpoints import (
     CACHE_SETTINGS_FIELDS,
@@ -160,8 +161,12 @@ class CacheSettingsManager:
         import json
 
         try:
-            cache_config = await CacheConfigRepository(prisma_client).table.find_unique(
-                where={"id": "cache_config"}
+            cache_config = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: CacheConfigRepository(prisma_client).table.find_unique(
+                    where={"id": "cache_config"}
+                ),
+                reason="init_cache_settings_in_db_lookup_failure",
             )
             if cache_config is not None and cache_config.cache_settings:
                 # Parse cache settings JSON

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -1850,11 +1850,10 @@ async def prepare_key_update_data(
 
     if "budget_duration" in non_default_values:
         budget_duration = non_default_values.pop("budget_duration")
-        if (
-            budget_duration
-            and (isinstance(budget_duration, str))
-            and len(budget_duration) > 0
-        ):
+        if budget_duration is None:
+            non_default_values["budget_duration"] = None
+            non_default_values["budget_reset_at"] = None
+        elif isinstance(budget_duration, str) and len(budget_duration) > 0:
             from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 
             key_reset_at = get_budget_reset_time(budget_duration=budget_duration)
@@ -2518,7 +2517,7 @@ async def update_key_fn(  # noqa: PLR0915
                 },
             )
 
-        data_json: dict = data.model_dump(exclude_unset=True, exclude_none=True)
+        data_json: dict = data.model_dump(exclude_unset=True)
         key = data_json.pop("key")
 
         # get the row from db
@@ -2588,6 +2587,17 @@ async def update_key_fn(  # noqa: PLR0915
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        if data.spend is not None:
+            try:
+                from litellm.proxy.proxy_server import _invalidate_spend_counter
+
+                token_to_invalidate = _hash_token_if_needed(key)
+                await _invalidate_spend_counter(
+                    counter_key=f"spend:key:{token_to_invalidate}"
+                )
+            except Exception:
+                pass
+
         asyncio.create_task(
             KeyManagementEventHooks.async_key_updated_hook(
                 data=data,
@@ -4775,6 +4785,13 @@ async def reset_key_spend_fn(
             proxy_logging_obj=proxy_logging_obj,
         )
 
+        try:
+            from litellm.proxy.proxy_server import _invalidate_spend_counter
+
+            await _invalidate_spend_counter(counter_key=f"spend:key:{hashed_api_key}")
+        except Exception:
+            pass
+
         max_budget = updated_key.max_budget
         budget_reset_at = updated_key.budget_reset_at
 

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -21,7 +21,7 @@ import json
 import os
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, Iterable, List, Literal, Optional
+from typing import Any, Dict, Iterable, List, Literal, Optional, Set
 
 from fastapi import (
     APIRouter,
@@ -1722,11 +1722,13 @@ if MCP_AVAILABLE:
                     status_code=status.HTTP_403_FORBIDDEN,
                     detail={"error": f"Access denied to MCP server {server_id}"},
                 )
-            allowed_server_ids = (
-                await global_mcp_server_manager.get_allowed_mcp_servers(
-                    user_api_key_dict
+            allowed_server_ids: Set[str] = set()
+            for auth_context in await build_effective_auth_contexts(user_api_key_dict):
+                allowed_server_ids.update(
+                    await global_mcp_server_manager.get_allowed_mcp_servers(
+                        auth_context
+                    )
                 )
-            )
             if server.server_id not in allowed_server_ids:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -357,6 +357,42 @@ class TeamMemberBudgetHandler:
         data_dict.pop("team_member_rpm_limit", None)
         data_dict.pop("team_member_tpm_limit", None)
 
+    @staticmethod
+    async def clear_team_member_budget_fields(
+        team_table: LiteLLM_TeamTable,
+        user_api_key_dict: "UserAPIKeyAuth",
+        updated_kv: dict,
+        explicitly_set_fields: set,
+    ) -> dict:
+        """Clear explicitly-nulled fields on the team member budget row."""
+        from litellm.proxy._types import BudgetNewRequest
+        from litellm.proxy.management_endpoints.budget_management_endpoints import (
+            update_budget,
+        )
+
+        if team_table.metadata is None:
+            team_table.metadata = {}
+
+        team_member_budget_id = team_table.metadata.get("team_member_budget_id")
+        if team_member_budget_id is not None and isinstance(team_member_budget_id, str):
+            budget_request = BudgetNewRequest(budget_id=team_member_budget_id)
+            if "team_member_budget" in explicitly_set_fields:
+                budget_request.max_budget = None
+            if "team_member_budget_duration" in explicitly_set_fields:
+                budget_request.budget_duration = None
+                budget_request.budget_reset_at = None
+            if "team_member_rpm_limit" in explicitly_set_fields:
+                budget_request.rpm_limit = None
+            if "team_member_tpm_limit" in explicitly_set_fields:
+                budget_request.tpm_limit = None
+            await update_budget(
+                budget_obj=budget_request,
+                user_api_key_dict=user_api_key_dict,
+            )
+
+        TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
+        return updated_kv
+
     @staticmethod
     async def backfill_team_member_budget_entries(
         team_id: str,
@@ -779,55 +815,40 @@ async def _check_user_team_limits(
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
     user_api_key_cache: Any,
-    existing_team_max_budget: Optional[float] = None,
 ) -> None:
     """
-    Check user team limits for standalone teams (not org-scoped).
+    Enforce the caller's personal limits when CREATING a standalone team.
 
-    This validates:
-    - Team budget vs user's max_budget
-    - Team models vs user's allowed models
+    This validates the requested team budget / models / tpm / rpm against the
+    caller's own limits, so a non-admin user cannot mint a brand-new team that
+    is richer than themselves.
 
-    Should only be called for standalone teams (when organization_id is None).
-    For org-scoped teams, use _check_org_team_limits() instead.
-
-    `existing_team_max_budget` is the team's current `max_budget` on the
-    /team/update path. When the incoming `max_budget` is unchanged or lower
-    than the team's current budget, the personal-budget comparison is skipped
-    so a team admin can edit other fields (e.g. tpm_limit, team name) without
-    being blocked by a budget the team already has. The UI sends the full team
-    object on every update, so the unchanged `max_budget` would otherwise fail.
+    Only used by /team/new for standalone teams (organization_id is None).
+    /team/update does NOT call this — an existing team's admin is already
+    authorized via _verify_team_access() and is not gated by their personal
+    wallet. Org-scoped teams use _check_org_team_limits() instead.
     """
     # Validate team budget against user's max_budget
     if data.max_budget is not None and user_api_key_dict.user_id is not None:
-        # On /team/update, allow unchanged or lower budgets without checking
-        # the caller's personal max_budget. Only increases above the team's
-        # current budget are validated against the user's personal limit.
-        budget_unchanged_or_lower = (
-            existing_team_max_budget is not None
-            and data.max_budget <= existing_team_max_budget
+        user_obj = await get_user_object(
+            user_id=user_api_key_dict.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
         )
 
-        if not budget_unchanged_or_lower:
-            user_obj = await get_user_object(
-                user_id=user_api_key_dict.user_id,
-                prisma_client=prisma_client,
-                user_api_key_cache=user_api_key_cache,
-                user_id_upsert=False,
+        if (
+            user_obj is not None
+            and user_obj.max_budget is not None
+            and data.max_budget > user_obj.max_budget
+        ):
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": f"max budget higher than user max. User max budget={user_obj.max_budget}. User role={user_api_key_dict.user_role}"
+                },
             )
 
-            if (
-                user_obj is not None
-                and user_obj.max_budget is not None
-                and data.max_budget > user_obj.max_budget
-            ):
-                raise HTTPException(
-                    status_code=400,
-                    detail={
-                        "error": f"max budget higher than user max. User max budget={user_obj.max_budget}. User role={user_api_key_dict.user_role}"
-                    },
-                )
-
     # Validate team models against user's allowed models
     if data.models is not None and len(user_api_key_dict.models) > 0:
         for m in data.models:
@@ -865,6 +886,45 @@ async def _check_user_team_limits(
         )
 
 
+def _check_team_budget_update_authority(
+    data: UpdateTeamRequest,
+    user_api_key_dict: UserAPIKeyAuth,
+    existing_team_max_budget: Optional[float],
+) -> None:
+    """
+    Restrict who can grow a standalone team's spend ceiling on /team/update.
+
+    A team admin (already authorized via _verify_team_access) may keep or lower
+    the team budget, but only a proxy admin may grow it - by raising max_budget
+    above the team's current value or by removing the cap (setting it to None).
+    Setting a finite budget on a team that has no cap is a restriction and is
+    allowed. Org-scoped teams are governed by _check_org_team_limits().
+    """
+    if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return
+    if existing_team_max_budget is None:
+        return
+
+    budget_explicitly_set = "max_budget" in (
+        getattr(data, "model_fields_set", None) or set()
+    )
+    if budget_explicitly_set and data.max_budget is None:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": f"Only a proxy admin can remove a team's max_budget. Team's current max_budget={existing_team_max_budget}."
+            },
+        )
+
+    if data.max_budget is not None and data.max_budget > existing_team_max_budget:
+        raise HTTPException(
+            status_code=403,
+            detail={
+                "error": f"Only a proxy admin can raise a team's max_budget. Team's current max_budget={existing_team_max_budget}, requested={data.max_budget}."
+            },
+        )
+
+
 #### TEAM MANAGEMENT ####
 @router.post(
     "/team/new",
@@ -1827,22 +1887,14 @@ async def update_team(  # noqa: PLR0915
                     prisma_client=prisma_client,
                 )
 
-        # Check user limits for standalone teams (not org-scoped)
-        # Skip for PROXY_ADMIN users
-        if (
-            user_api_key_dict.user_role is None
-            or user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
-        ):
-            # Only validate user budget/models for standalone teams
-            # For org-scoped teams, validation is done by _check_org_team_limits() above
-            if org_id_to_check is None:
-                await _check_user_team_limits(
-                    data=data,
-                    user_api_key_dict=user_api_key_dict,
-                    prisma_client=prisma_client,
-                    user_api_key_cache=user_api_key_cache,
-                    existing_team_max_budget=existing_team_row.max_budget,
-                )
+        # Only a proxy admin may grow a standalone team's spend ceiling.
+        # Org-scoped teams are validated by _check_org_team_limits() above.
+        if org_id_to_check is None:
+            _check_team_budget_update_authority(
+                data=data,
+                user_api_key_dict=user_api_key_dict,
+                existing_team_max_budget=existing_team_row.max_budget,
+            )
 
         updated_kv = data.json(exclude_unset=True)
 
@@ -1856,11 +1908,25 @@ async def update_team(  # noqa: PLR0915
         # Check budget_duration and budget_reset_at
         _set_budget_reset_at(data, updated_kv)
 
-        if TeamMemberBudgetHandler.should_create_budget(
-            team_member_budget=data.team_member_budget,
-            team_member_rpm_limit=data.team_member_rpm_limit,
-            team_member_tpm_limit=data.team_member_tpm_limit,
-            team_member_budget_duration=data.team_member_budget_duration,
+        _team_member_fields_in_request = {
+            field
+            for field in [
+                "team_member_budget",
+                "team_member_rpm_limit",
+                "team_member_tpm_limit",
+                "team_member_budget_duration",
+            ]
+            if field in updated_kv
+        }
+
+        if (
+            _team_member_fields_in_request
+            and TeamMemberBudgetHandler.should_create_budget(
+                team_member_budget=data.team_member_budget,
+                team_member_rpm_limit=data.team_member_rpm_limit,
+                team_member_tpm_limit=data.team_member_tpm_limit,
+                team_member_budget_duration=data.team_member_budget_duration,
+            )
         ):
             updated_kv = await TeamMemberBudgetHandler.upsert_team_member_budget_table(
                 team_table=existing_team_row,
@@ -1883,6 +1949,13 @@ async def update_team(  # noqa: PLR0915
                     team_member_budget_id=_backfill_budget_id,
                     prisma_client=prisma_client,
                 )
+        elif _team_member_fields_in_request:
+            updated_kv = await TeamMemberBudgetHandler.clear_team_member_budget_fields(
+                team_table=existing_team_row,
+                user_api_key_dict=user_api_key_dict,
+                updated_kv=updated_kv,
+                explicitly_set_fields=_team_member_fields_in_request,
+            )
         else:
             TeamMemberBudgetHandler._clean_team_member_fields(updated_kv)
 
@@ -1971,6 +2044,8 @@ def _set_budget_reset_at(data: UpdateTeamRequest, updated_kv: dict) -> None:
 
         reset_at = get_budget_reset_time(budget_duration=data.budget_duration)
         updated_kv["budget_reset_at"] = reset_at
+    elif "budget_duration" in updated_kv and updated_kv["budget_duration"] is None:
+        updated_kv["budget_reset_at"] = None
 
     if data.budget_limits is not None and len(data.budget_limits) > 0:
         from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -426,12 +426,7 @@ async def mistral_proxy_route(
     )
 
     ## check for streaming
-    is_streaming_request = False
-    # anthropic is streaming when 'stream' = True is in the body
-    if request.method == "POST":
-        _request_body = await request.json()
-        if _request_body.get("stream"):
-            is_streaming_request = True
+    is_streaming_request = await is_streaming_request_fn(request)
 
     ## CREATE PASS-THROUGH
     endpoint_func = create_pass_through_route(

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -5,7 +5,7 @@ Handles cost tracking and logging for OpenAI passthrough endpoints, specifically
 """
 
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import List, Optional, Tuple, Union
 from urllib.parse import urlparse
 
 import httpx
@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import (
 )
 from litellm.llms.openai.openai import OpenAIConfig
 from litellm.llms.openai.openai import OpenAIConfig as OpenAIConfigType
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough_logging_handler import (
     BasePassthroughLoggingHandler,
@@ -29,6 +30,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     EndpointType,
     PassthroughStandardLoggingPayload,
 )
+from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.utils import ImageResponse, LlmProviders, PassthroughCallTypes
 from litellm.utils import ModelResponse, TextCompletionResponse
 
@@ -236,6 +238,42 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
             )
             return 0.0
 
+    @staticmethod
+    def _build_responses_api_response_and_cost(
+        model: str,
+        httpx_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        custom_llm_provider: str,
+    ) -> Tuple[ResponsesAPIResponse, float]:
+        """Transform a Responses API raw response into a ResponsesAPIResponse
+        and compute its cost.
+
+        The Responses API has a different on-the-wire shape from chat
+        completions (`output: [...]` instead of `choices: [...]`), so the
+        chat-completions `transform_response` raises KeyError 'choices' on
+        a Responses payload. Use the dedicated Responses-API transformer
+        (`OpenAIResponsesAPIConfig.transform_response_api_response`) here.
+
+        Returns (litellm_model_response, response_cost) — symmetric with the
+        chat-completions branch which produces the same two values inline,
+        and analogous to the image branches' `_calculate_image_*_cost` helpers
+        (which return cost only because the image-response object is trivial
+        to build inline; the Responses payload needs a real transformer).
+        """
+        responses_config = OpenAIResponsesAPIConfig()
+        litellm_model_response = responses_config.transform_response_api_response(
+            model=model,
+            raw_response=httpx_response,
+            logging_obj=logging_obj,
+        )
+        response_cost = litellm.completion_cost(
+            completion_response=litellm_model_response,
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            call_type="responses",
+        )
+        return litellm_model_response, response_cost
+
     @staticmethod
     def openai_passthrough_handler(  # noqa: PLR0915
         httpx_response: httpx.Response,
@@ -301,7 +339,12 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
         try:
             response_cost = 0.0
             litellm_model_response: Optional[
-                Union[ModelResponse, TextCompletionResponse, ImageResponse]
+                Union[
+                    ModelResponse,
+                    TextCompletionResponse,
+                    ImageResponse,
+                    ResponsesAPIResponse,
+                ]
             ] = None
             handler_instance = OpenAIPassthroughLoggingHandler()
 
@@ -384,29 +427,18 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     litellm_model_response._hidden_params = {}
                 litellm_model_response._hidden_params["response_cost"] = response_cost
             elif is_responses:
-                # Handle responses API cost calculation
-                provider_config = handler_instance.get_provider_config(model=model)
-                existing_litellm_params = kwargs.get("litellm_params", {}) or {}
-                litellm_model_response = provider_config.transform_response(
-                    raw_response=httpx_response,
-                    model_response=litellm.ModelResponse(),
+                # Responses-API cost tracking — see
+                # `_build_responses_api_response_and_cost` for why this needs
+                # a dedicated transformer (the chat-completions transform
+                # crashes on the Responses payload shape).
+                (
+                    litellm_model_response,
+                    response_cost,
+                ) = OpenAIPassthroughLoggingHandler._build_responses_api_response_and_cost(
                     model=model,
-                    messages=request_body.get("messages", []),
+                    httpx_response=httpx_response,
                     logging_obj=logging_obj,
-                    optional_params=request_body.get("optional_params", {}),
-                    api_key="",
-                    request_data=request_body,
-                    encoding=litellm.encoding,
-                    json_mode=False,
-                    litellm_params=existing_litellm_params,
-                )
-
-                # Calculate cost using LiteLLM's cost calculator with responses call type
-                response_cost = litellm.completion_cost(
-                    completion_response=litellm_model_response,
-                    model=model,
                     custom_llm_provider=custom_llm_provider,
-                    call_type="responses",
                 )
 
             # Update kwargs with cost information

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -1099,6 +1099,20 @@ async def pass_through_request(  # noqa: PLR0915
                     status_code=e.response.status_code, detail=await e.response.aread()
                 )
 
+            # Call response headers hook for streaming pass-through
+            _response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
+                headers=response.headers,
+                litellm_call_id=litellm_call_id,
+            )
+            callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+                data=_parsed_body or {},
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_headers=dict(request.headers),
+            )
+            if callback_headers:
+                _response_headers.update(callback_headers)
+
             return StreamingResponse(
                 PassThroughStreamingHandler.chunk_processor(
                     response=response,
@@ -1109,10 +1123,7 @@ async def pass_through_request(  # noqa: PLR0915
                     passthrough_success_handler_obj=pass_through_endpoint_logging,
                     url_route=str(url),
                 ),
-                headers=HttpPassThroughEndpointHelpers.get_response_headers(
-                    headers=response.headers,
-                    litellm_call_id=litellm_call_id,
-                ),
+                headers=_response_headers,
                 status_code=response.status_code,
             )
 
@@ -1151,6 +1162,20 @@ async def pass_through_request(  # noqa: PLR0915
                     status_code=e.response.status_code, detail=await e.response.aread()
                 )
 
+            # Call response headers hook for detected streaming pass-through
+            _response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
+                headers=response.headers,
+                litellm_call_id=litellm_call_id,
+            )
+            callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+                data=_parsed_body or {},
+                user_api_key_dict=user_api_key_dict,
+                response=response,
+                request_headers=dict(request.headers),
+            )
+            if callback_headers:
+                _response_headers.update(callback_headers)
+
             return StreamingResponse(
                 PassThroughStreamingHandler.chunk_processor(
                     response=response,
@@ -1161,10 +1186,7 @@ async def pass_through_request(  # noqa: PLR0915
                     passthrough_success_handler_obj=pass_through_endpoint_logging,
                     url_route=str(url),
                 ),
-                headers=HttpPassThroughEndpointHelpers.get_response_headers(
-                    headers=response.headers,
-                    litellm_call_id=litellm_call_id,
-                ),
+                headers=_response_headers,
                 status_code=response.status_code,
             )
 
@@ -1303,6 +1325,16 @@ async def pass_through_request(  # noqa: PLR0915
             api_base=str(url._uri_reference),
         )
 
+        # Call response headers hook
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=_parsed_body or {},
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            custom_headers.update(callback_headers)
+
         response_headers = HttpPassThroughEndpointHelpers.get_response_headers(
             headers=response.headers,
             custom_headers=custom_headers,

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -458,7 +458,16 @@ class PassThroughEndpointLogging:
         return False
 
     def _is_supported_openai_endpoint(self, url_route: str) -> bool:
-        """Check if the OpenAI endpoint is supported by the passthrough logging handler."""
+        """Check if the OpenAI endpoint is supported by the passthrough logging handler.
+
+        The Responses API route is included because
+        `openai_passthrough_handler` has a dedicated `elif is_responses:`
+        branch that knows how to extract usage + cost from the
+        Responses-API on-the-wire shape. Without including it here, the
+        outer dispatch filters Responses calls out before reaching the
+        handler — the inner branch is then unreachable and Responses
+        calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
+        """
         from .llm_provider_handlers.openai_passthrough_logging_handler import (
             OpenAIPassthroughLoggingHandler,
         )
@@ -469,6 +478,7 @@ class PassThroughEndpointLogging:
                 url_route
             )
             or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
         )
 
     def _set_cost_per_request(

litellm/proxy/proxy_cli.py

@@ -44,15 +44,19 @@ def _build_db_connection_url_params(
     pool_timeout: Optional[Union[int, float]],
     connect_timeout: Optional[Union[int, float]] = None,
     socket_timeout: Optional[Union[int, float]] = None,
+    disable_prepared_statements: bool = False,
     extra_params: Optional[dict] = None,
 ) -> dict:
     """Build the Prisma DATABASE_URL query params controlling connection pool behavior.
 
     `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same
     name (https://www.prisma.io/docs/orm/overview/databases/postgresql) and are
-    omitted when None so Prisma's defaults apply. `extra_params` is an
-    untyped passthrough — keys it provides win over the named arguments above,
-    so it can be used to override any default we set here.
+    omitted when None so Prisma's defaults apply. `disable_prepared_statements`
+    sets `pgbouncer=true`, which makes Prisma stop using server-side prepared
+    statements (pgbouncer transaction-pool compatible; also sidesteps the
+    "cached plan must not change result type" error during rolling migrations).
+    `extra_params` is an untyped passthrough — keys it provides win over the
+    named arguments above, so it can be used to override any default we set here.
     """
     params: dict = {
         "connection_limit": connection_limit,
@@ -63,6 +67,8 @@ def _build_db_connection_url_params(
         params["connect_timeout"] = connect_timeout
     if socket_timeout is not None:
         params["socket_timeout"] = socket_timeout
+    if disable_prepared_statements:
+        params["pgbouncer"] = "true"
     if extra_params:
         params.update(extra_params)
     return params
@@ -555,6 +561,7 @@ class ProxyInitializationHelpers:
 
 
 @click.command()
+@click.argument("cli_args", nargs=-1)
 @click.option(
     "--host", default="0.0.0.0", help="Host for the server to listen on.", envvar="HOST"
 )
@@ -808,6 +815,7 @@ class ProxyInitializationHelpers:
     help="Enable uvicorn hot reload (dev only). Also reloads when the --config YAML file changes. Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.",
 )
 def run_server(  # noqa: PLR0915
+    cli_args,
     host,
     port,
     api_base,
@@ -854,6 +862,20 @@ def run_server(  # noqa: PLR0915
     use_v2_migration_resolver: bool,
     reload: bool,
 ):
+    if cli_args:
+        if cli_args == ("xai-oauth", "login"):
+            from litellm.llms.xai.oauth import XAIOAuthAuthenticator
+
+            authenticator = XAIOAuthAuthenticator()
+            auth_data = authenticator.login()
+            click.echo(
+                f"xAI OAuth login successful. Credentials saved to {authenticator.auth_file}."
+            )
+            if auth_data.get("expires_at"):
+                click.echo(f"Access token expires at {auth_data['expires_at']}.")
+            return
+        raise click.UsageError(f"Unknown command: {' '.join(cli_args)}")
+
     if setup:
         from litellm.setup_wizard import run_setup_wizard
 
@@ -947,6 +969,7 @@ def run_server(  # noqa: PLR0915
         db_connection_timeout: Optional[Union[int, float]] = 60
         db_connect_timeout: Optional[Union[int, float]] = None
         db_socket_timeout: Optional[Union[int, float]] = None
+        db_disable_prepared_statements: bool = False
         db_extra_connection_params: Optional[dict] = None
         general_settings = {}
         ### GET DB TOKEN FOR IAM AUTH ###
@@ -1067,6 +1090,17 @@ def run_server(  # noqa: PLR0915
                 )
             db_connect_timeout = general_settings.get("database_connect_timeout")
             db_socket_timeout = general_settings.get("database_socket_timeout")
+            _disable_prepared_statements = general_settings.get(
+                "database_disable_prepared_statements", False
+            )
+            if isinstance(_disable_prepared_statements, str):
+                from litellm.secret_managers.main import str_to_bool
+
+                db_disable_prepared_statements = (
+                    str_to_bool(_disable_prepared_statements) is True
+                )
+            else:
+                db_disable_prepared_statements = bool(_disable_prepared_statements)
             db_extra_connection_params = general_settings.get(
                 "database_extra_connection_params"
             )
@@ -1114,6 +1148,7 @@ def run_server(  # noqa: PLR0915
                     pool_timeout=db_connection_timeout,
                     connect_timeout=db_connect_timeout,
                     socket_timeout=db_socket_timeout,
+                    disable_prepared_statements=db_disable_prepared_statements,
                     extra_params=db_extra_connection_params,
                 )
                 if os.getenv("DATABASE_URL", None) is not None:

litellm/proxy/proxy_server.py

@@ -311,7 +311,10 @@ from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.container_endpoints.endpoints import router as container_router
 from litellm.proxy.credential_endpoints.endpoints import router as credential_router
 from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup
-from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
+from litellm.proxy.db.exception_handler import (
+    PrismaDBExceptionHandler,
+    call_with_db_reconnect_retry,
+)
 from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
 from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
@@ -2263,7 +2266,7 @@ async def _reconcile_budget_reservation_for_counter_update(
         )
     except Exception:
         verbose_proxy_logger.warning(
-            "Failed to reconcile budget reservation after persisted spend; invalidating reserved counters and continuing",
+            "Failed to reconcile budget reservation after persisted spend; invalidating reserved counters and falling back to direct increment",
             exc_info=True,
         )
         try:
@@ -2274,6 +2277,7 @@ async def _reconcile_budget_reservation_for_counter_update(
             verbose_proxy_logger.exception(
                 "Failed to invalidate reserved counters after reservation reconciliation failed"
             )
+        return set()
     return reserved_counter_keys
 
 
@@ -4075,6 +4079,7 @@ class ProxyConfig:
                         premium_user=premium_user,
                         config_file_path=config_file_path,
                         litellm_settings=litellm_settings,
+                        callback_specific_params=callback_settings,
                     )
 
                 elif key == "model_group_settings":
@@ -4089,6 +4094,8 @@ class ProxyConfig:
                     verbose_proxy_logger.debug(
                         f"litellm.post_call_rules: {litellm.post_call_rules}"
                     )
+                elif key == "max_budget":
+                    litellm.max_budget = float(value)
                 elif key == "max_internal_user_budget":
                     litellm.max_internal_user_budget = float(value)  # type: ignore
                 elif key == "default_max_internal_user_budget":
@@ -5982,8 +5989,12 @@ class ProxyConfig:
         """
 
         try:
-            sso_settings = await SSOConfigRepository(prisma_client).table.find_unique(
-                where={"id": "sso_config"}
+            sso_settings = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: SSOConfigRepository(prisma_client).table.find_unique(
+                    where={"id": "sso_config"}
+                ),
+                reason="init_sso_settings_in_db_lookup_failure",
             )
             if sso_settings is not None:
                 sso_settings.sso_settings.pop("role_mappings", None)
@@ -6018,9 +6029,13 @@ class ProxyConfig:
         )
 
         try:
-            db_record = await ConfigOverridesRepository(
-                prisma_client
-            ).table.find_unique(where={"config_type": "hashicorp_vault"})
+            db_record = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
+                    where={"config_type": "hashicorp_vault"}
+                ),
+                reason="init_hashicorp_vault_config_override_lookup_failure",
+            )
 
             if db_record is None or db_record.config_value is None:
                 if self._last_hashicorp_vault_config is not None:
@@ -9202,6 +9217,16 @@ async def audio_speech(
             hidden_params=hidden_params,
         )
 
+        # Call response headers hook (matches audio_transcription behavior)
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            custom_headers.update(callback_headers)
+
         # Determine media type based on model type
         media_type = "audio/mpeg"  # Default for OpenAI TTS
         request_model = data.get("model", "")
@@ -11126,6 +11151,22 @@ async def _get_caller_byok_team_scope(
     return set(user_row.teams or [])
 
 
+def _byok_row_outside_caller_teams(
+    model_info_dict: Dict[str, Any], allowed_team_ids: Optional[Set[str]]
+) -> bool:
+    """Whether a team BYOK row belongs to a team the caller is not a member of.
+
+    `team_id` is only set on team BYOK rows; non-team rows fall through
+    unaffected. `allowed_team_ids is None` means no scoping (e.g. admins).
+    """
+    if allowed_team_ids is None:
+        return False
+    team_id = model_info_dict.get("team_id")
+    if team_id is None:
+        return False
+    return team_id not in allowed_team_ids
+
+
 # Hard cap on rows the DB-side BYOK search may pull when results need to be
 # sorted across the full match set. Without this, an authenticated caller
 # can hit `/v2/model/info?search=<broad>&sortBy=<field>` and force the
@@ -11247,15 +11288,7 @@ async def _apply_search_filter_to_models(
     )
 
     def _is_byok_outside_caller_teams(model_info_dict: Dict[str, Any]) -> bool:
-        # `team_id` is only set on team BYOK rows. Non-team rows fall
-        # through unaffected — they are gated by other paths (router
-        # membership, direct_access, include_team_models).
-        if allowed_team_ids is None:
-            return False
-        team_id = model_info_dict.get("team_id")
-        if team_id is None:
-            return False
-        return team_id not in allowed_team_ids
+        return _byok_row_outside_caller_teams(model_info_dict, allowed_team_ids)
 
     def _model_matches_search(m: Dict[str, Any]) -> bool:
         # Team BYOK models persist an internal `model_name`
@@ -12395,6 +12428,72 @@ async def model_metrics_exceptions(
     return {"data": response, "exception_types": list(exception_types)}
 
 
+def _deployment_matches_allowed_model_names(
+    model: Dict[str, Any], allowed_model_names: Set[str]
+) -> bool:
+    """Match a router deployment against allowed public model names.
+
+    Team-scoped rows store an internal routing key in ``model_name``; callers
+    with key/team restrictions still refer to the public name in
+    ``model_info.team_public_model_name``.
+    """
+    if model.get("model_name") in allowed_model_names:
+        return True
+    model_info = model.get("model_info")
+    if not isinstance(model_info, dict):
+        return False
+    team_public_model_name = model_info.get("team_public_model_name")
+    return (
+        isinstance(team_public_model_name, str)
+        and team_public_model_name in allowed_model_names
+    )
+
+
+def _get_v1_model_info_allowed_model_names(
+    user_api_key_dict: UserAPIKeyAuth,
+    llm_router: Router,
+) -> Optional[Set[str]]:
+    """Return key/team allowlisted public model names, or None if unrestricted."""
+    model_access_groups = llm_router.get_model_access_groups()
+    proxy_model_list = llm_router.get_model_names()
+    key_models = get_key_models(
+        user_api_key_dict=user_api_key_dict,
+        proxy_model_list=proxy_model_list,
+        model_access_groups=model_access_groups,
+    )
+    team_models = get_team_models(
+        team_models=user_api_key_dict.team_models,
+        proxy_model_list=proxy_model_list,
+        model_access_groups=model_access_groups,
+    )
+    if not key_models and not team_models:
+        return None
+    return set(
+        get_complete_model_list(
+            key_models=key_models,
+            team_models=team_models,
+            proxy_model_list=proxy_model_list,
+            user_model=user_model,
+            infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
+            llm_router=llm_router,
+            return_wildcard_routes=False,
+        )
+    )
+
+
+def _filter_v1_model_info_deployments(
+    all_models: List[dict],
+    allowed_model_names: Optional[Set[str]],
+) -> List[dict]:
+    if allowed_model_names is None:
+        return all_models
+    return [
+        model
+        for model in all_models
+        if _deployment_matches_allowed_model_names(model, allowed_model_names)
+    ]
+
+
 def _translate_model_name_for_response(model: dict) -> dict:
     """For team-scoped DB rows, replace `model_name` with the public name
     in `model_info.team_public_model_name` before returning. The DB column
@@ -12564,49 +12663,42 @@ async def model_info_v1(  # noqa: PLR0915
         )
         return {"data": [_deployment_info_dict]}
 
-    all_models: List[dict] = []
-    model_access_groups: Dict[str, List[str]] = defaultdict(list)
-    ## CHECK IF MODEL RESTRICTIONS ARE SET AT KEY/TEAM LEVEL ##
-    if llm_router is None:
-        proxy_model_list = []
-    else:
-        proxy_model_list = llm_router.get_model_names()
-        model_access_groups = llm_router.get_model_access_groups()
-    key_models = get_key_models(
+    # Return router deployments (same source as /v2/model/info), not wildcard-
+    # expanded model names from get_complete_model_list(). Team-scoped rows
+    # use internal routing keys (model_name_{team_id}_{uuid}) and were omitted
+    # when v1 resolved models only via public model_name strings.
+    all_models: List[dict] = copy.deepcopy(llm_router.model_list)
+    allowed_model_names = _get_v1_model_info_allowed_model_names(
         user_api_key_dict=user_api_key_dict,
-        proxy_model_list=proxy_model_list,
-        model_access_groups=model_access_groups,
-    )
-    team_models = get_team_models(
-        team_models=user_api_key_dict.team_models,
-        proxy_model_list=proxy_model_list,
-        model_access_groups=model_access_groups,
-    )
-    all_models_str = get_complete_model_list(
-        key_models=key_models,
-        team_models=team_models,
-        proxy_model_list=proxy_model_list,
-        user_model=user_model,
-        infer_model_from_keys=general_settings.get("infer_model_from_keys", False),
         llm_router=llm_router,
     )
 
-    if len(all_models_str) > 0:
-        _relevant_models = []
-        for model in all_models_str:
-            router_models = llm_router.get_model_list(model_name=model)
-            if router_models is not None:
-                _relevant_models.extend(router_models)
-        if llm_model_list is not None:
-            all_models = copy.deepcopy(_relevant_models)  # type: ignore
-        else:
-            all_models = []
+    all_models = _filter_v1_model_info_deployments(
+        all_models=all_models,
+        allowed_model_names=allowed_model_names,
+    )
 
-    # Reassign each entry: _get_proxy_model_info returns a (possibly new)
-    # dict via _translate_model_name_for_response, which does NOT mutate in
-    # place. Binding only the loop variable would drop the public-name swap
-    # for team-scoped rows and leak the internal routing key (#28382).
-    all_models = [_get_proxy_model_info(model=model) for model in all_models]
+    # Team BYOK deployments carry an internal routing key and other teams'
+    # public name/team_id/api_base; drop the ones the caller cannot access so
+    # listing the full router model_list does not leak cross-team metadata.
+    allowed_team_ids = await _get_caller_byok_team_scope(
+        user_api_key_dict=user_api_key_dict,
+        prisma_client=prisma_client,
+    )
+    all_models = [
+        model
+        for model in all_models
+        if not _byok_row_outside_caller_teams(
+            model.get("model_info") or {}, allowed_team_ids
+        )
+    ]
+
+    all_models = [
+        _translate_model_name_for_response(
+            _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router)
+        )
+        for model in all_models
+    ]
 
     verbose_proxy_logger.debug("all_models: %s", all_models)
     return {"data": all_models}

litellm/proxy/search_endpoints/search_tool_registry.py

@@ -7,6 +7,7 @@ from typing import List, Optional
 
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.table_repositories import SearchToolsRepository
 from litellm.types.search import SearchTool
@@ -180,10 +181,12 @@ class SearchToolRegistry:
             List of search tool configurations
         """
         try:
-            search_tools_from_db = await SearchToolsRepository(
-                prisma_client
-            ).table.find_many(
-                order={"created_at": "desc"},
+            search_tools_from_db = await call_with_db_reconnect_retry(
+                prisma_client,
+                lambda: SearchToolsRepository(prisma_client).table.find_many(
+                    order={"created_at": "desc"},
+                ),
+                reason="get_all_search_tools_from_db_lookup_failure",
             )
 
             search_tools: List[SearchTool] = []

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -3405,7 +3405,7 @@ async def ui_view_session_spend_logs(
                 session_id, status, mcp_namespaced_tool_name, agent_id
             FROM "LiteLLM_SpendLogs"
             WHERE session_id = $1
-            ORDER BY "startTime" ASC
+            ORDER BY "startTime" DESC
             LIMIT $2 OFFSET $3
         """
         result = await prisma_client.db.query_raw(

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -178,6 +178,11 @@ class UISettings(BaseModel):
         description="If true, org admins cannot generate API keys via /key/generate.",
     )
 
+    disable_ui_nudges: bool = Field(
+        default=False,
+        description="If true, suppresses in-product UI nudges (survey and Claude Code feedback popups) for all users.",
+    )
+
 
 class UISettingsResponse(SettingsResponse):
     """Response model for UI settings"""
@@ -201,6 +206,7 @@ ALLOWED_UI_SETTINGS_FIELDS = {
     "scope_user_search_to_org",
     "disable_custom_api_keys",
     "disable_key_generate_for_org_admin",
+    "disable_ui_nudges",
 }
 
 # Flags that must be synced from the persisted UISettings into

litellm/proxy/utils.py

@@ -2499,7 +2499,8 @@ class ProxyLogging:
         )
 
         return {
-            "custom_llm_provider": hidden_params.get("custom_llm_provider"),
+            "custom_llm_provider": hidden_params.get("custom_llm_provider")
+            or getattr(response, "custom_llm_provider", None),
             "model_info": model_info,
             "api_base": hidden_params.get("api_base"),
             "model_id": hidden_params.get("model_id"),
@@ -3291,40 +3292,49 @@ class PrismaClient:
         self, sql_query: str, *args
     ) -> Optional[dict]:
         """
-        Execute a query with automatic fallback for PostgreSQL cached plan errors.
+        Execute a query, recovering once from PostgreSQL's "cached plan must not
+        change result type" error.
 
-        This handles the "cached plan must not change result type" error that occurs
-        during rolling deployments when schema changes are applied while old pods
-        still have cached query plans expecting the old schema.
+        That error surfaces during rolling deployments when a schema change
+        invalidates the prepared-statement plans that pooled connections still
+        hold. Clearing only the server-side plans with DEALLOCATE ALL makes
+        things worse: Prisma's query engine keeps a per-connection client-side
+        cache of prepared-statement names, so once the server drops a plan the
+        engine re-sends a name PostgreSQL no longer recognizes and the
+        connection breaks with `prepared statement "sN" does not exist`. With a
+        small pool that connection stays poisoned and every auth lookup fails.
 
-        Args:
-            sql_query: SQL query string to execute
+        Recreating the Prisma client kills the engine subprocess and drops the
+        server-side plans and the engine's client-side name cache together, so
+        the retried query is prepared fresh. We reconnect through
+        `attempt_db_reconnect`, which is singleflight: when a schema change
+        poisons every pooled connection at once, the first cached-plan error
+        recreates the client and the concurrent waiters reuse that single
+        recreate instead of racing to kill each other's fresh engine. We then
+        retry the identical query exactly once.
 
-        Returns:
-            Query result or None
+        The retry reuses the original query byte-for-byte. Mutating the SQL
+        (e.g. injecting a unique comment) would defeat PostgreSQL's plan cache,
+        forcing a fresh plan on every request and pegging the database CPU.
 
-        Raises:
-            Original exception if not a cached plan error
+        If the reconnect is skipped because a recent reconnect is still within
+        its cooldown, the retry runs against the same connection and may fail
+        again; the get_data backoff decorator re-runs the lookup and a later
+        attempt reconnects once the cooldown elapses.
         """
         try:
             return await self.db.query_first(sql_query, *args)
         except Exception as e:
-            error_str = str(e)
-            if "cached plan must not change result type" in error_str:
-                # Force PostgreSQL to re-plan by invalidating the cache
-                # Add a unique comment to make the query different
-                sql_query_retry = sql_query.replace(
-                    "SELECT",
-                    f"SELECT /* cache_invalidated_{int(time.time() * 1000)} */",
-                )
-                verbose_proxy_logger.warning(
-                    "PostgreSQL cached plan error detected for token lookup, "
-                    "retrying with fresh plan. This may occur during rolling deployments "
-                    "when schema changes are applied."
-                )
-                return await self.db.query_first(sql_query_retry, *args)
-            else:
+            if "cached plan must not change result type" not in str(e):
                 raise
+            verbose_proxy_logger.warning(
+                "PostgreSQL cached plan error detected for token lookup; "
+                "recreating the database connection and retrying with the same "
+                "query. This may occur during rolling deployments when schema "
+                "changes are applied."
+            )
+            await self.attempt_db_reconnect(reason="postgres_cached_plan_error")
+            return await self.db.query_first(sql_query, *args)
 
     @backoff.on_exception(
         backoff.expo,

litellm/router.py

@@ -1658,6 +1658,67 @@ class Router:
                     f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys."
                 )
 
+    def _add_encrypted_content_affinity_check(
+        self, enable_global_affinity: bool
+    ) -> None:
+        from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
+            EncryptedContentAffinityCheck,
+        )
+
+        def _move_before_deployment_affinity(
+            callback_list: List[Any],
+            callback_to_move: EncryptedContentAffinityCheck,
+        ) -> None:
+            if callback_to_move not in callback_list:
+                return
+            callback_list.remove(callback_to_move)
+            insert_index = next(
+                (
+                    idx
+                    for idx, callback in enumerate(callback_list)
+                    if isinstance(callback, DeploymentAffinityCheck)
+                ),
+                len(callback_list),
+            )
+            callback_list.insert(insert_index, callback_to_move)
+
+        if (
+            enable_global_affinity
+            or EncryptedContentAffinityCheck.has_model_group_affinity_enabled(
+                self.model_group_affinity_config
+            )
+        ):
+            if self.optional_callbacks is None:
+                self.optional_callbacks = []
+
+            existing_ec_callback: Optional[EncryptedContentAffinityCheck] = None
+            for cb in self.optional_callbacks:
+                if isinstance(cb, EncryptedContentAffinityCheck):
+                    existing_ec_callback = cb
+                    break
+
+            if existing_ec_callback is not None:
+                existing_ec_callback.router = self
+                existing_ec_callback.enable_global_affinity = (
+                    existing_ec_callback.enable_global_affinity
+                    or enable_global_affinity
+                )
+                existing_ec_callback.model_group_affinity_config = (
+                    self.model_group_affinity_config or {}
+                )
+                ec_callback = existing_ec_callback
+            else:
+                ec_callback = EncryptedContentAffinityCheck(
+                    router=self,
+                    enable_global_affinity=enable_global_affinity,
+                    model_group_affinity_config=self.model_group_affinity_config,
+                )
+                self.optional_callbacks.append(ec_callback)
+                litellm.logging_callback_manager.add_litellm_callback(ec_callback)
+
+            _move_before_deployment_affinity(self.optional_callbacks, ec_callback)
+            _move_before_deployment_affinity(litellm.callbacks, ec_callback)
+
     def add_optional_pre_call_checks(
         self, optional_pre_call_checks: Optional[OptionalPreCallChecks]
     ):
@@ -1721,22 +1782,11 @@ class Router:
         # ---------------------------------------------------------------------
         # Encrypted content affinity
         # ---------------------------------------------------------------------
-        if "encrypted_content_affinity" in optional_pre_call_checks:
-            from litellm.router_utils.pre_call_checks.encrypted_content_affinity_check import (
-                EncryptedContentAffinityCheck,
+        self._add_encrypted_content_affinity_check(
+            enable_global_affinity=(
+                "encrypted_content_affinity" in optional_pre_call_checks
             )
-
-            if self.optional_callbacks is None:
-                self.optional_callbacks = []
-
-            already_registered = any(
-                isinstance(cb, EncryptedContentAffinityCheck)
-                for cb in self.optional_callbacks
-            )
-            if not already_registered:
-                ec_callback = EncryptedContentAffinityCheck(router=self)
-                self.optional_callbacks.append(ec_callback)
-                litellm.logging_callback_manager.add_litellm_callback(ec_callback)
+        )
 
         # ---------------------------------------------------------------------
         # Remaining optional pre-call checks
@@ -7738,6 +7788,39 @@ class Router:
 
         return hash_object.hexdigest()
 
+    @staticmethod
+    def _inherit_builtin_cache_pricing(
+        model_info: dict, backend_model: str, custom_llm_provider: Optional[str]
+    ) -> None:
+        """Fill missing cache pricing on a custom-priced deployment entry from
+        the backend model's built-in cost map entry, so a deployment that
+        only spells out ``input_cost_per_token``/``output_cost_per_token``
+        does not silently bill cache_read/cache_creation at 0.
+
+        User-specified cache fields always win; only ``None``/missing entries
+        are inherited. No-op when the backend model has no canonical entry.
+        """
+        cache_fields = (
+            "cache_creation_input_token_cost",
+            "cache_creation_input_token_cost_above_1hr",
+            "cache_creation_input_token_cost_above_200k_tokens",
+            "cache_read_input_token_cost",
+            "cache_read_input_token_cost_above_200k_tokens",
+        )
+        if all(model_info.get(f) is not None for f in cache_fields):
+            return
+        try:
+            backend_info = litellm.get_model_info(
+                model=backend_model, custom_llm_provider=custom_llm_provider
+            )
+        except Exception:
+            return
+        for field in cache_fields:
+            if model_info.get(field) is None:
+                backend_value = backend_info.get(field)
+                if backend_value is not None:
+                    model_info[field] = backend_value
+
     def _create_deployment(
         self,
         deployment_info: dict,
@@ -7766,6 +7849,13 @@ class Router:
                 if deployment.litellm_params.get(field) is not None:
                     _model_info[field] = deployment.litellm_params[field]
 
+            if _model_info.get("input_cost_per_token") is not None:
+                Router._inherit_builtin_cache_pricing(
+                    model_info=_model_info,
+                    backend_model=deployment.litellm_params.model,
+                    custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+                )
+
             ## REGISTER MODEL INFO IN LITELLM MODEL COST MAP
             model_id = deployment.model_info.id
             if model_id is not None:
@@ -8471,6 +8561,13 @@ class Router:
                     credential_values.get("api_key")
                     or deployment.litellm_params.api_key
                 )
+                if api_key is None:
+                    verbose_router_logger.debug(
+                        "Skipping pass-through credential setup for deployment model=%s, custom_llm_provider=%s; no api_key set. Providers like bedrock resolve credentials at request time.",
+                        model,
+                        custom_llm_provider,
+                    )
+                    return
                 passthrough_endpoint_router.set_pass_through_credentials(
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
@@ -8505,6 +8602,13 @@ class Router:
             if field_value is not None:
                 _model_info_dict[field] = field_value
 
+        if _model_info_dict.get("input_cost_per_token") is not None:
+            Router._inherit_builtin_cache_pricing(
+                model_info=_model_info_dict,
+                backend_model=deployment.litellm_params.model,
+                custom_llm_provider=deployment.litellm_params.custom_llm_provider,
+            )
+
         # Register custom pricing in litellm.model_cost.
         # Mirrors _create_deployment() logic to ensure dynamically-added deployments
         # (e.g., loaded from DB) also have their custom pricing registered.

litellm/router_utils/pre_call_checks/deployment_affinity_check.py

@@ -39,7 +39,12 @@ class DeploymentAffinityCheck(CustomLogger):
 
     CACHE_KEY_PREFIX = "deployment_affinity:v1"
     VALID_FLAGS = frozenset(
-        {"deployment_affinity", "responses_api_deployment_check", "session_affinity"}
+        {
+            "deployment_affinity",
+            "responses_api_deployment_check",
+            "session_affinity",
+            "encrypted_content_affinity",
+        }
     )
 
     def __init__(

litellm/router_utils/pre_call_checks/encrypted_content_affinity_check.py

@@ -37,7 +37,7 @@ Safe to enable globally:
 """
 
 import time
-from typing import TYPE_CHECKING, Any, List, Optional, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, cast
 
 import httpx
 
@@ -64,17 +64,45 @@ class EncryptedContentAffinityCheck(CustomLogger):
     The ``model_id`` is decoded directly from the litellm-encoded item IDs –
     no caching or TTL management needed.
 
-    Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])``.
+    Wired via ``Router(optional_pre_call_checks=["encrypted_content_affinity"])`` or
+    per-model group ``model_group_affinity_config``.
     """
 
-    def __init__(self, router: Optional["Router"] = None) -> None:
+    def __init__(
+        self,
+        router: Optional["Router"] = None,
+        enable_global_affinity: bool = True,
+        model_group_affinity_config: Optional[Dict[str, List[str]]] = None,
+    ) -> None:
         super().__init__()
         self.router = router
+        self.enable_global_affinity = enable_global_affinity
+        self.model_group_affinity_config: Dict[str, List[str]] = (
+            model_group_affinity_config or {}
+        )
 
     # ------------------------------------------------------------------
     # Helpers
     # ------------------------------------------------------------------
 
+    @staticmethod
+    def has_model_group_affinity_enabled(
+        model_group_affinity_config: Optional[Dict[str, List[str]]],
+    ) -> bool:
+        if not model_group_affinity_config:
+            return False
+
+        return any(
+            "encrypted_content_affinity" in checks
+            for checks in model_group_affinity_config.values()
+        )
+
+    def _is_enabled_for_model_group(self, model_group: str) -> bool:
+        group_checks = self.model_group_affinity_config.get(model_group)
+        return self.enable_global_affinity or (
+            group_checks is not None and "encrypted_content_affinity" in group_checks
+        )
+
     @staticmethod
     def _extract_model_id_from_input(request_input: Any) -> Optional[str]:
         """
@@ -213,6 +241,8 @@ class EncryptedContentAffinityCheck(CustomLogger):
         """
         request_kwargs = request_kwargs or {}
         typed_healthy_deployments = cast(List[dict], healthy_deployments)
+        if not self._is_enabled_for_model_group(model):
+            return typed_healthy_deployments
 
         # Signal to the response post-processor that encrypted item IDs should be
         # encoded in the output of this request.  Only set the flag when

litellm/setup_wizard.py

@@ -52,11 +52,12 @@ PROVIDERS: List[Dict] = [
     {
         "id": "anthropic",
         "name": "Anthropic",
-        "description": "Claude Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5",
+        "description": "Claude Fable 5, Opus 4.8, Opus 4.7, Opus 4.6, Sonnet 4.6, Haiku 4.5",
         "env_key": "ANTHROPIC_API_KEY",
         "key_hint": "sk-ant-...",
         "test_model": "claude-haiku-4-5-20251001",
         "models": [
+            "claude-fable-5",
             "claude-opus-4-8",
             "claude-opus-4-7",
             "claude-opus-4-6",

litellm/types/caching.py

@@ -118,4 +118,5 @@ class CachedEmbedding(TypedDict):
     index: Optional[int]
     object: Optional[str]
     model: Optional[str]
+    prompt_tokens: Optional[int]
     prompt_tokens_details: Optional[dict]

litellm/types/router.py

@@ -178,6 +178,7 @@ class CredentialLiteLLMParams(BaseModel):
     aws_secret_access_key: Optional[str] = None
     aws_region_name: Optional[str] = None
     aws_bedrock_runtime_endpoint: Optional[str] = None
+    aws_bedrock_project_id: Optional[str] = None
     ## IBM WATSONX ##
     watsonx_region_name: Optional[str] = None
 
@@ -220,6 +221,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     use_in_pass_through: Optional[bool] = False
     use_litellm_proxy: Optional[bool] = False
     use_chat_completions_api: Optional[bool] = None
+    use_xai_oauth: Optional[bool] = Field(
+        default=False,
+        description="Use stored xAI OAuth credentials when no xAI API key is configured.",
+    )
     model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
     merge_reasoning_content_in_choices: Optional[bool] = False
     model_info: Optional[Dict] = None
@@ -360,6 +365,7 @@ class LiteLLMParamsTypedDict(TypedDict, total=False):
     aws_access_key_id: Optional[str]
     aws_secret_access_key: Optional[str]
     aws_region_name: Optional[str]
+    aws_bedrock_project_id: Optional[str]
     ## AWS S3 VECTORS ##
     vector_bucket_name: Optional[str]
     index_name: Optional[str]

litellm/types/utils.py

@@ -197,6 +197,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     ]  # OpenAI priority service tier pricing
     cache_read_input_token_cost_above_200k_tokens: Optional[float]
     cache_read_input_token_cost_above_272k_tokens: Optional[float]
+    cache_read_input_token_cost_above_512k_tokens: Optional[float]
     input_cost_per_character: Optional[float]  # only for vertex ai models
     input_cost_per_audio_token: Optional[float]
     input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models
@@ -206,6 +207,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     input_cost_per_token_above_272k_tokens: Optional[
         float
     ]  # GPT-5.4/5.4-pro: prompts >272K priced at 2x input
+    input_cost_per_token_above_512k_tokens: Optional[
+        float
+    ]  # MiniMax-M3: prompts >512K priced at 2x input
     input_cost_per_character_above_128k_tokens: Optional[
         float
     ]  # only for vertex ai models
@@ -239,6 +243,9 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     output_cost_per_token_above_272k_tokens: Optional[
         float
     ]  # GPT-5.4/5.4-pro: prompts >272K priced at 1.5x output
+    output_cost_per_token_above_512k_tokens: Optional[
+        float
+    ]  # MiniMax-M3: prompts >512K priced at 2x output
     output_cost_per_character_above_128k_tokens: Optional[
         float
     ]  # only for vertex ai models
@@ -3217,6 +3224,7 @@ all_litellm_params = (
         "search_tool_name",
         "order",
         "enable_json_schema_validation",
+        "use_xai_oauth",
     ]
     + list(StandardCallbackDynamicParams.__annotations__.keys())
     + list(CustomPricingLiteLLMParams.model_fields.keys())

litellm/utils.py

@@ -2887,6 +2887,61 @@ def _convert_stringified_numbers(value):
     return value
 
 
+_BEDROCK_REGION_PREFIXES = (
+    "us.",
+    "eu.",
+    "apac.",
+    "jp.",
+    "au.",
+    "us-gov.",
+    "global.",
+    "ap-northeast-1.",
+)
+
+_CACHE_PRICING_FIELDS = (
+    "cache_creation_input_token_cost",
+    "cache_creation_input_token_cost_above_1hr",
+    "cache_creation_input_token_cost_above_200k_tokens",
+    "cache_read_input_token_cost",
+    "cache_read_input_token_cost_above_200k_tokens",
+)
+
+
+def _resolve_builtin_model_cost_entry(
+    key: str, provider: str
+) -> Optional[Dict[str, Any]]:
+    """Best-effort lookup of a built-in ``model_cost`` entry for a custom key
+    whose shape ``get_model_info`` cannot resolve (double provider prefixes
+    like ``bedrock/bedrock/us.anthropic.claude-sonnet-4-6`` or region aliases).
+
+    Returns a copy of the matching entry so the caller can inherit its defaults
+    (most importantly cache pricing) without mutating the shared built-in.
+    Returns ``None`` when no safe match exists.
+    """
+    candidates: List[str] = []
+    segments = key.split("/")
+    idx = 0
+    while idx < len(segments) - 1 and segments[idx] in LlmProvidersSet:
+        idx += 1
+        candidates.append("/".join(segments[idx:]))
+
+    base = candidates[-1] if candidates else key
+    for region_prefix in _BEDROCK_REGION_PREFIXES:
+        if base.startswith(region_prefix):
+            candidates.append(base[len(region_prefix) :])
+
+    if provider:
+        stripped = _strip_model_name(model=base, custom_llm_provider=provider)
+        if stripped != base:
+            candidates.append(stripped)
+
+    for candidate in candidates:
+        entry = litellm.model_cost.get(candidate)
+        if entry is not None and entry.get("litellm_provider") is not None:
+            return dict(entry)
+    return None
+
+
 def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
     """
     Register new / Override existing models (and their pricing) to specific providers.
@@ -2933,6 +2988,26 @@ def register_model(model_cost: Union[str, dict]):  # noqa: PLR0915
             except Exception:
                 existing_model = {}
                 model_cost_key = key
+                builtin_entry = _resolve_builtin_model_cost_entry(
+                    key=_key_str, provider=provider
+                )
+                if builtin_entry is not None:
+                    for field in _CACHE_PRICING_FIELDS:
+                        if (
+                            value.get(field) is None
+                            and builtin_entry.get(field) is not None
+                        ):
+                            existing_model[field] = builtin_entry[field]
+                elif (
+                    value.get("cache_creation_input_token_cost") is None
+                    and value.get("cache_read_input_token_cost") is None
+                ):
+                    verbose_logger.warning(
+                        f"register_model: model={key} not in built-in cost map and no "
+                        "prefix/region variant matched; cache cost fields will default "
+                        "to 0. To track cache cost, add cache_creation_input_token_cost "
+                        "and cache_read_input_token_cost to model_info"
+                    )
         # ``get_model_info`` returns ``litellm_provider: None`` when the
         # provider is unknown (e.g. custom deployments registered via
         # ``Router.add_deployment``). Persisting that None into
@@ -3762,6 +3837,10 @@ class PreProcessNonDefaultParams:
         additional_endpoint_specific_params: List[str],
     ) -> dict:
         for k, v in special_params.items():
+            if k == "aws_bedrock_project_id":
+                # sent as a request header (read from litellm_params by the
+                # bedrock-mantle configs), never as a request body field
+                continue
             if k.startswith("aws_") and (
                 custom_llm_provider != "bedrock"
                 and not custom_llm_provider.startswith("sagemaker")
@@ -5775,6 +5854,7 @@ def _get_model_info_helper(  # noqa: PLR0915
         ]
         split_model = potential_model_names["split_model"]
         custom_llm_provider = potential_model_names["custom_llm_provider"]
+        model_cost_custom_llm_provider = custom_llm_provider
         #########################
         provider_config: Optional[BaseLLMModelInfo] = None
         if custom_llm_provider and custom_llm_provider in LlmProvidersSet:
@@ -5840,7 +5920,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                 key = _matched_key
                 _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                 if not _check_provider_match(
-                    model_info=_model_info, custom_llm_provider=custom_llm_provider
+                    model_info=_model_info,
+                    custom_llm_provider=model_cost_custom_llm_provider,
                 ):
                     _model_info = None
             if _model_info is None:
@@ -5849,7 +5930,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5858,7 +5940,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5867,7 +5950,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
             if _model_info is None:
@@ -5876,7 +5960,8 @@ def _get_model_info_helper(  # noqa: PLR0915
                     key = _matched_key
                     _model_info = _get_model_info_from_model_cost(key=cast(str, key))
                     if not _check_provider_match(
-                        model_info=_model_info, custom_llm_provider=custom_llm_provider
+                        model_info=_model_info,
+                        custom_llm_provider=model_cost_custom_llm_provider,
                     ):
                         _model_info = None
 
@@ -5884,7 +5969,6 @@ def _get_model_info_helper(  # noqa: PLR0915
                 raise ValueError(
                     "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
                 )
-
             _input_cost_per_token: Optional[float] = _model_info.get(
                 "input_cost_per_token"
             )
@@ -5936,6 +6020,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 cache_read_input_token_cost_above_272k_tokens=_model_info.get(
                     "cache_read_input_token_cost_above_272k_tokens", None
                 ),
+                cache_read_input_token_cost_above_512k_tokens=_model_info.get(
+                    "cache_read_input_token_cost_above_512k_tokens", None
+                ),
                 cache_read_input_token_cost_flex=_model_info.get(
                     "cache_read_input_token_cost_flex", None
                 ),
@@ -5957,6 +6044,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 input_cost_per_token_above_272k_tokens=_model_info.get(
                     "input_cost_per_token_above_272k_tokens", None
                 ),
+                input_cost_per_token_above_512k_tokens=_model_info.get(
+                    "input_cost_per_token_above_512k_tokens", None
+                ),
                 input_cost_per_query=_model_info.get("input_cost_per_query", None),
                 input_cost_per_second=_model_info.get("input_cost_per_second", None),
                 input_cost_per_audio_token=_model_info.get(
@@ -6012,6 +6102,9 @@ def _get_model_info_helper(  # noqa: PLR0915
                 output_cost_per_token_above_272k_tokens=_model_info.get(
                     "output_cost_per_token_above_272k_tokens", None
                 ),
+                output_cost_per_token_above_512k_tokens=_model_info.get(
+                    "output_cost_per_token_above_512k_tokens", None
+                ),
                 output_cost_per_second=_model_info.get("output_cost_per_second", None),
                 output_cost_per_second_1080p=_model_info.get(
                     "output_cost_per_second_1080p", None
@@ -8922,14 +9015,33 @@ class ProviderConfigManager:
         elif litellm.LlmProviders.HOSTED_VLLM == provider:
             return litellm.HostedVLLMResponsesAPIConfig()
         elif litellm.LlmProviders.BEDROCK_MANTLE == provider:
-            # Only OpenAI gpt frontier models (gpt-5.x, and future gpt-6 etc.) are
-            # served on the /openai/v1/responses path. gpt-oss and every non-OpenAI
-            # model on Mantle (nvidia, mistral, google, zai, ...) are chat-completions
-            # only and 400 on that path, so they fall through to None to keep the
-            # chat-completions emulation (see litellm/responses/main.py "config is None").
-            model_lower = model.lower() if model else ""
-            if "openai.gpt-" in model_lower and "gpt-oss" not in model_lower:
-                return litellm.BedrockMantleResponsesAPIConfig()
+            # Mantle serves Responses on two upstream paths. A model takes the
+            # /openai/v1/responses path when its price-map entry declares
+            # use_openai_responses_path (data-driven, so a non-gpt-named frontier
+            # model can be onboarded by JSON alone), or, as a fallback needing no
+            # price-map entry, when its name matches the openai.gpt- frontier
+            # convention (minus gpt-oss) -- this keeps a future gpt-6 routing
+            # correctly before its entry loads. Any other model declared
+            # mode=responses takes the standard /v1/responses path. Everything
+            # else returns None and keeps the chat-completions emulation (see
+            # responses/main.py "config is None").
+            if not model:
+                return None
+            model_lower = model.lower()
+            entry = litellm.model_cost.get(f"bedrock_mantle/{model}", {})
+            on_openai_path = entry.get("use_openai_responses_path") is True
+            name_is_frontier = (
+                "openai.gpt-" in model_lower and "gpt-oss" not in model_lower
+            )
+            if on_openai_path or name_is_frontier:
+                return litellm.BedrockMantleResponsesAPIConfig(use_openai_path=True)
+            try:
+                if get_model_info(model, "bedrock_mantle").get("mode") == "responses":
+                    return litellm.BedrockMantleResponsesAPIConfig(
+                        use_openai_path=False
+                    )
+            except Exception:
+                pass
             return None
         return None
 

model_prices_and_context_window.json

@@ -1156,6 +1156,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1202,6 +1203,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1233,6 +1235,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1264,6 +1267,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1295,6 +1299,139 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "global.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "us.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_native_structured_output": true,
+        "supports_max_reasoning_effort": true,
+        "supports_output_config": true,
+        "bedrock_output_config_effort_ceiling": "xhigh"
+    },
+    "eu.anthropic.claude-fable-5": {
+        "cache_creation_input_token_cost": 1.375e-05,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-05,
+        "cache_read_input_token_cost": 1.1e-06,
+        "input_cost_per_token": 1.1e-05,
+        "litellm_provider": "bedrock_converse",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1327,6 +1464,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1359,6 +1497,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1391,6 +1530,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1423,6 +1563,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1455,6 +1596,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -1485,6 +1627,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2208,6 +2351,37 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "azure_ai/claude-fable-5": {
+        "input_cost_per_token": 1e-05,
+        "output_cost_per_token": 5e-05,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -2237,6 +2411,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10170,6 +10345,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10204,6 +10380,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -10214,6 +10391,40 @@
         },
         "supports_output_config": true
     },
+    "claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "anthropic",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "provider_specific_entry": {
+            "us": 1.1
+        },
+        "supports_output_config": true
+    },
     "claude-opus-4-8": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_creation_input_token_cost_above_1hr": 1e-05,
@@ -10238,6 +10449,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -24180,9 +24392,12 @@
         "max_output_tokens": 8192
     },
     "minimax/MiniMax-M3": {
-        "input_cost_per_token": 6e-07,
-        "output_cost_per_token": 2.4e-06,
-        "cache_read_input_token_cost": 1.2e-07,
+        "input_cost_per_token": 3e-07,
+        "input_cost_per_token_above_512k_tokens": 6e-07,
+        "output_cost_per_token": 1.2e-06,
+        "output_cost_per_token_above_512k_tokens": 2.4e-06,
+        "cache_read_input_token_cost": 6e-08,
+        "cache_read_input_token_cost_above_512k_tokens": 1.2e-07,
         "litellm_provider": "minimax",
         "mode": "chat",
         "supports_function_calling": true,
@@ -24191,7 +24406,7 @@
         "supports_reasoning": true,
         "supports_system_messages": true,
         "supports_vision": true,
-        "max_input_tokens": 512000,
+        "max_input_tokens": 1000000,
         "max_output_tokens": 128000
     },
     "mistral.devstral-2-123b": {
@@ -34044,6 +34259,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34072,6 +34288,67 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true
+    },
+    "vertex_ai/claude-fable-5@default": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_1hr": 2e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "vertex_ai-anthropic_models",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supports_adaptive_thinking": true,
+        "supports_assistant_prefill": false,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34101,6 +34378,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -34130,6 +34408,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true,
@@ -41410,6 +41689,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41429,6 +41709,7 @@
         "max_output_tokens": 128000,
         "max_tokens": 128000,
         "mode": "responses",
+        "use_openai_responses_path": true,
         "supported_endpoints": ["/v1/responses"],
         "supported_modalities": ["text", "image"],
         "supported_output_modalities": ["text"],
@@ -41902,5 +42183,164 @@
     "source": "https://soniox.com/pricing",
     "supported_endpoints": ["/v1/audio/transcriptions"],
     "supports_audio_input": true
+  },
+  "tensormesh/Qwen/Qwen3.5-397B-A17B-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 6e-07,
+    "output_cost_per_token": 3.6e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 4.5e-07,
+    "output_cost_per_token": 1.8e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/Qwen/Qwen3.6-27B-FP8": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 3.2e-07,
+    "output_cost_per_token": 3.2e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 262144,
+    "max_output_tokens": 262144,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/lukealonso/GLM-5.1-NVFP4-MTP": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-06,
+    "output_cost_per_token": 4.4e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 202752,
+    "max_output_tokens": 202752,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/deepseek-ai/DeepSeek-V4-Flash": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-07,
+    "output_cost_per_token": 2.8e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/moonshotai/Kimi-K2.6": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 9.6e-07,
+    "output_cost_per_token": 4e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/MiniMaxAI/MiniMax-M2.5": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 3e-07,
+    "output_cost_per_token": 1.2e-06,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 196608,
+    "max_output_tokens": 196608,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/google/gemma-4-31B-it": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.4e-07,
+    "output_cost_per_token": 5.6e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 32768,
+    "max_output_tokens": 32768,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/openai/gpt-oss-120b": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 1.5e-07,
+    "output_cost_per_token": 6e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 131072,
+    "max_output_tokens": 131072,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
+  },
+  "tensormesh/openai/gpt-oss-20b": {
+    "litellm_provider": "tensormesh",
+    "mode": "chat",
+    "input_cost_per_token": 7e-08,
+    "output_cost_per_token": 2.8e-07,
+    "cache_read_input_token_cost": 0,
+    "max_input_tokens": 131072,
+    "max_output_tokens": 131072,
+    "supports_function_calling": true,
+    "supports_tool_choice": true,
+    "supports_response_schema": true,
+    "supports_prompt_caching": true,
+    "supports_system_messages": true,
+    "supports_reasoning": true,
+    "source": "https://serverless.tensormesh.ai/v1/models/openrouter"
   }
  }

packaging/homebrew/README.md

@@ -0,0 +1,27 @@
+# Homebrew formula for the `lite` CLI
+
+[`lite.rb`](./lite.rb) is the canonical source for the Homebrew formula that installs the thin LiteLLM CLI (`litellm[cli]`). It lives here so it is versioned with the code, but Homebrew serves formulae from a tap, so it has to be published to the `BerriAI/homebrew-litellm` tap to be installable.
+
+Once published, end users install with
+
+```shell
+brew install BerriAI/litellm/lite
+```
+
+which gives them the `lite` command (`lite login`, `lite claude`, `lite models list`, ...) without the proxy server runtime. For the full proxy server, they keep using pip/uv with `litellm[proxy]` or the Docker image.
+
+## Why a tap and not homebrew-core
+
+The formula builds the published `litellm` sdist with the `cli` extra and resolves that extra's dependencies from PyPI at build time. homebrew-core forbids network access during `install` and would require every transitive dependency declared as a pinned `resource`, regenerated on each release. For a fast-moving CLI that tradeoff is not worth it, so this stays a tap formula.
+
+## Release runbook
+
+The formula can only point at a published artifact, so it activates with the first `litellm` release that ships the `cli` extra (added in [pyproject.toml](../../pyproject.toml)).
+
+1. Cut a `litellm` release whose `pyproject.toml` includes the `cli` extra and confirm it is on PyPI.
+2. Fetch the sdist URL and checksum for that version: `curl -fsSL https://pypi.org/pypi/litellm/<version>/json | jq -r '.urls[] | select(.packagetype=="sdist") | "\(.url)\n\(.digests.sha256)"'`
+3. Set `url` and `sha256` in `lite.rb` to those values; `version` is parsed from `url`.
+4. Copy `lite.rb` into the tap repo under `Formula/lite.rb`, then run `brew install --build-from-source ./Formula/lite.rb` and `brew test lite` to verify a clean build and that `lite --help` works.
+5. Commit and push to `BerriAI/homebrew-litellm`.
+
+Keep `lite.rb` here in sync with the tap copy so the in-repo formula stays the source of truth.

packaging/homebrew/lite.rb

@@ -0,0 +1,33 @@
+# Homebrew formula for the thin LiteLLM `lite` CLI (litellm[cli]).
+#
+# Ships in the BerriAI/homebrew-litellm tap, not homebrew-core: it builds the
+# published litellm sdist with the `cli` extra into a dedicated virtualenv and
+# pulls the extra's deps from PyPI. That is the low-maintenance path for a
+# fast-moving Python CLI; the resource-stanza alternative would need every
+# transitive dep re-pinned with a fresh sha256 on each release.
+#
+# RELEASE STEP (see README.md in this directory): point `url` + `sha256` at the
+# PyPI sdist of the first litellm version that ships the `cli` extra. `version`
+# is parsed from `url`, and the build installs exactly that version, so the three
+# stay in lockstep automatically.
+class Lite < Formula
+  include Language::Python::Virtualenv
+
+  desc "Thin client for the LiteLLM proxy: lite login, lite claude/codex/opencode"
+  homepage "https://docs.litellm.ai/docs/proxy/management_cli"
+  url "https://files.pythonhosted.org/packages/source/l/litellm/litellm-REPLACE_AT_RELEASE.tar.gz"
+  sha256 "REPLACE_AT_RELEASE"
+  license "MIT"
+
+  depends_on "python@3.13"
+
+  def install
+    virtualenv_create(libexec, "python3.13")
+    system libexec/"bin/pip", "install", "#{buildpath}[cli]"
+    bin.install_symlink libexec/"bin/lite"
+  end
+
+  test do
+    assert_match "login", shell_output("#{bin}/lite --help")
+  end
+end

pyproject.toml

@@ -71,6 +71,14 @@ proxy = [
     "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'",
     "pydantic-settings>=2.14.1,<3.0",
 ]
+# Thin client install for the `lite` CLI on developer laptops. The CLI's heavy
+# imports (fastapi, cryptography, ...) are all guarded, so it runs on the base
+# SDK plus just these three; none of the server runtime in `proxy` is pulled in.
+cli = [
+    "rich>=13.9.4,<14.0",
+    "pyyaml>=6.0.3,<7.0",
+    "requests>=2.32.0,<3.0",
+]
 extra_proxy = [
     "prisma>=0.11.0,<1.0",
     "azure-identity>=1.25.2,<2.0",
@@ -132,6 +140,7 @@ proxy-runtime = [
 
 [project.scripts]
 litellm = "litellm:run_server"
+lite = "litellm.proxy.client.cli:cli"
 litellm-proxy = "litellm.proxy.client.cli:cli"
 
 [dependency-groups]

scripts/install-cli.sh

@@ -0,0 +1,128 @@
+#!/usr/bin/env bash
+# LiteLLM CLI Installer (the thin `lite` client)
+# Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install-cli.sh | sh
+#
+# Installs only litellm[cli]: the `lite` command for authenticating to a LiteLLM
+# proxy and running coding agents (lite claude / codex / opencode) through it.
+# None of the proxy server runtime is pulled in. To run a proxy server instead,
+# use scripts/install.sh, which installs litellm[proxy].
+#
+# Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible
+# Python itself (honouring litellm's requires-python), downloading a managed one
+# when the host has no suitable interpreter.
+#
+# NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
+# ignores the shebang when invoked as `sh` and does not support `pipefail`).
+set -eu
+
+# NOTE: before merging, this must stay as "litellm[cli]" to install from PyPI.
+LITELLM_PACKAGE="litellm[cli]"
+UV_VERSION="0.10.9"
+
+# ── colours ────────────────────────────────────────────────────────────────
+if [ -t 1 ]; then
+  BOLD='\033[1m'
+  GREEN='\033[38;2;78;186;101m'
+  GREY='\033[38;2;153;153;153m'
+  RESET='\033[0m'
+else
+  BOLD='' GREEN='' GREY='' RESET=''
+fi
+
+info()    { printf "${GREY}  %s${RESET}\n" "$*"; }
+success() { printf "${GREEN}  ✔ %s${RESET}\n" "$*"; }
+header()  { printf "${BOLD}  %s${RESET}\n" "$*"; }
+die()     { printf "\n  Error: %s\n\n" "$*" >&2; exit 1; }
+
+# ── banner ─────────────────────────────────────────────────────────────────
+echo ""
+cat << 'EOF'
+  ██╗     ██╗████████╗███████╗
+  ██║     ██║╚══██╔══╝██╔════╝
+  ██║     ██║   ██║   █████╗
+  ██║     ██║   ██║   ██╔══╝
+  ███████╗██║   ██║   ███████╗
+  ╚══════╝╚═╝   ╚═╝   ╚══════╝
+EOF
+printf "  ${BOLD}LiteLLM CLI Installer${RESET}  ${GREY}the thin 'lite' client for your proxy${RESET}\n\n"
+
+# ── OS detection ───────────────────────────────────────────────────────────
+OS="$(uname -s)"
+ARCH="$(uname -m)"
+
+case "$OS" in
+  Darwin)  PLATFORM="macOS ($ARCH)" ;;
+  Linux)   PLATFORM="Linux ($ARCH)" ;;
+  *)       die "Unsupported OS: $OS. LiteLLM supports macOS and Linux." ;;
+esac
+
+info "Platform: $PLATFORM"
+
+# ── uv detection / install ────────────────────────────────────────────────
+UV_BIN=""
+CURRENT_UV_VERSION=""
+for candidate in uv "$HOME/.local/bin/uv"; do
+  if command -v "$candidate" >/dev/null 2>&1; then
+    UV_BIN="$(command -v "$candidate")"
+    break
+  elif [ -x "$candidate" ]; then
+    UV_BIN="$candidate"
+    break
+  fi
+done
+
+if [ -n "$UV_BIN" ]; then
+  CURRENT_UV_VERSION="$("$UV_BIN" --version 2>/dev/null | awk '{print $2}' | head -1 || true)"
+fi
+
+if [ -z "$UV_BIN" ] || [ "${CURRENT_UV_VERSION:-}" != "$UV_VERSION" ]; then
+  header "Installing uv…"
+  if [ -n "${CURRENT_UV_VERSION:-}" ]; then
+    info "Upgrading uv from ${CURRENT_UV_VERSION} to ${UV_VERSION}"
+  fi
+  curl -LsSf "https://astral.sh/uv/${UV_VERSION}/install.sh" | env UV_NO_MODIFY_PATH=1 sh \
+    || die "uv installation failed. Try manually: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh"
+  UV_BIN="$HOME/.local/bin/uv"
+fi
+
+# ── install ────────────────────────────────────────────────────────────────
+# --python-preference system: reuse a compatible system Python when present,
+# otherwise download a managed one. Either way uv honours litellm's requires-python,
+# so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced.
+echo ""
+header "Installing litellm[cli]…"
+echo ""
+
+"$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \
+  || die "uv tool install failed. Try manually: $UV_BIN tool install '${LITELLM_PACKAGE}'"
+
+# ── find the lite binary installed by uv tool ──────────────────────────────
+SCRIPTS_DIR="$("$UV_BIN" tool dir --bin)"
+LITE_BIN="${SCRIPTS_DIR}/lite"
+
+if [ ! -x "$LITE_BIN" ]; then
+  die "lite binary not found after install. Try: $UV_BIN tool install '${LITELLM_PACKAGE}'"
+fi
+
+# ── success banner ─────────────────────────────────────────────────────────
+echo ""
+success "LiteLLM CLI installed"
+
+installed_ver="$("$LITE_BIN" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true)"
+[ -n "$installed_ver" ] && info "Version: $installed_ver"
+
+# ── PATH hint ──────────────────────────────────────────────────────────────
+if ! command -v lite >/dev/null 2>&1; then
+  info "Note: add lite to your PATH:  export PATH=\"\$PATH:${SCRIPTS_DIR}\""
+fi
+
+# ── next steps ─────────────────────────────────────────────────────────────
+echo ""
+header "Next steps:"
+echo ""
+info "  export LITELLM_PROXY_URL=https://your-proxy   # point at your gateway"
+info "  lite login                                    # authenticate via SSO"
+info "  lite claude                                   # run Claude Code through the proxy"
+echo ""
+info "Docs: https://docs.litellm.ai/docs/proxy/management_cli"
+echo ""

scripts/install.sh

@@ -2,13 +2,13 @@
 # LiteLLM Installer
 # Usage: curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh
 #
+# Needs only curl: uv is bootstrapped if missing, and uv provisions a compatible
+# Python itself (reusing a suitable system one, else downloading a managed build).
+#
 # NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian
 # ignores the shebang when invoked as `sh` and does not support `pipefail`).
 set -eu
 
-MIN_PYTHON_MAJOR=3
-MIN_PYTHON_MINOR=9
-
 # NOTE: before merging, this must stay as "litellm[proxy]" to install from PyPI.
 LITELLM_PACKAGE="litellm[proxy]"
 UV_VERSION="0.10.9"
@@ -52,27 +52,6 @@ esac
 
 info "Platform: $PLATFORM"
 
-# ── Python detection ───────────────────────────────────────────────────────
-PYTHON_BIN=""
-for candidate in python3 python; do
-  if command -v "$candidate" >/dev/null 2>&1; then
-    major="$("$candidate" -c 'import sys; print(sys.version_info.major)' 2>/dev/null || true)"
-    minor="$("$candidate" -c 'import sys; print(sys.version_info.minor)' 2>/dev/null || true)"
-    if [ "${major:-0}" -ge "$MIN_PYTHON_MAJOR" ] && [ "${minor:-0}" -ge "$MIN_PYTHON_MINOR" ]; then
-      PYTHON_BIN="$(command -v "$candidate")"
-      info "Python: $("$candidate" --version 2>&1)"
-      break
-    fi
-  fi
-done
-
-if [ -z "$PYTHON_BIN" ]; then
-  die "Python ${MIN_PYTHON_MAJOR}.${MIN_PYTHON_MINOR}+ is required but not found.
-  Install it from https://python.org/downloads or via your package manager:
-    macOS:  brew install python@3
-    Ubuntu: sudo apt install python3"
-fi
-
 # ── uv detection / install ────────────────────────────────────────────────
 UV_BIN=""
 CURRENT_UV_VERSION=""
@@ -105,15 +84,18 @@ echo ""
 header "Installing litellm[proxy]…"
 echo ""
 
-"$UV_BIN" tool install --python "$PYTHON_BIN" --force "${LITELLM_PACKAGE}" \
-  || die "uv tool install failed. Try manually: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'"
+# --python-preference system: reuse a compatible system Python when present,
+# otherwise download a managed one. Either way uv honours litellm's requires-python,
+# so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced.
+"$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \
+  || die "uv tool install failed. Try manually: $UV_BIN tool install '${LITELLM_PACKAGE}'"
 
 # ── find the litellm binary installed by uv tool ───────────────────────────
 SCRIPTS_DIR="$("$UV_BIN" tool dir --bin)"
 LITELLM_BIN="${SCRIPTS_DIR}/litellm"
 
 if [ ! -x "$LITELLM_BIN" ]; then
-  die "litellm binary not found after install. Try: $UV_BIN tool install --python '$PYTHON_BIN' '${LITELLM_PACKAGE}'"
+  die "litellm binary not found after install. Try: $UV_BIN tool install '${LITELLM_PACKAGE}'"
 fi
 
 # ── success banner ─────────────────────────────────────────────────────────

security.md

@@ -3,12 +3,20 @@
 
 ## Security Vulnerability Reporting Guidelines
 
+> [!WARNING]
+> Reports that do not include a video demonstrating the exploit will be closed without review. See [Reproduction Video Requirement](#reproduction-video-requirement) below.
+
 We value the security community's role in protecting our systems and users. To report a security vulnerability:
 
 - File a private vulnerability report on GitHub: [Report a vulnerability](https://github.com/BerriAI/litellm/security/advisories/new)
 - Include steps to reproduce the issue
+- Include a video or screen recording demonstrating the full exploit against a live LiteLLM instance, from initial access through to impact. A terminal recording (for example asciinema) is fine for CLI-only exploits.
 - Provide any relevant additional information
 
+### Reproduction Video Requirement
+
+A video demonstrating the exploit is required for every report. AI tools have made it easy to produce plausible-sounding vulnerability reports that do not reproduce in practice, and triaging them takes time away from real issues. Reports submitted without a working reproduction video will be closed without review. If you add a video to a closed report, we will reopen and triage it.
+
 ### Vulnerability Categories
 
 We classify vulnerabilities into the following categories:
@@ -38,7 +46,7 @@ We offer bounties for responsibly disclosed vulnerabilities based on severity:
 | **Medium** | N/A | P2 authenticated privilege escalation |
 | **Low** | N/A | Minor information disclosure, low-impact misconfigurations |
 
-To qualify for a bounty, reports must include clear reproduction steps and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days.
+To qualify for a bounty, reports must include clear reproduction steps, a reproduction video as described above, and must not involve systems or accounts you do not own. We review all submissions promptly and will follow up within 5 business days.
 
 ### Known Non-Issues
 

tests/llm_translation/reasoning_effort_grid/grid_spec.py

@@ -1,7 +1,6 @@
 from dataclasses import dataclass, field
 from typing import Dict, FrozenSet, List, Optional, Tuple
 
-
 OMIT = object()
 
 
@@ -136,6 +135,13 @@ _CAPS_NONE: FrozenSet[str] = frozenset()
 
 
 ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="claude-fable-5",
+        model="anthropic/claude-fable-5",
+        mode="adaptive",
+        required_env=_ANTHROPIC_REQ,
+        caps=_CAPS_XHIGH_MAX,
+    ),
     ModelEntry(
         alias="claude-opus-4-8",
         model="anthropic/claude-opus-4-8",
@@ -168,6 +174,19 @@ ANTHROPIC_DIRECT_MODELS: Tuple[ModelEntry, ...] = (
 
 
 AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="azure-claude-fable-5",
+        model="azure_ai/claude-fable-5",
+        mode="adaptive",
+        required_env=_AZURE_FOUNDRY_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        fail_reason=(
+            "claude-fable-5 has no deployment on the CI Microsoft Foundry "
+            "resource yet; Foundry returns DeploymentNotFound until someone "
+            "creates the fable-5 deployment, so this cell stays loud in CI. "
+            "Remove this fail_reason once the deployment exists."
+        ),
+    ),
     ModelEntry(
         alias="azure-claude-opus-4-8",
         model="azure_ai/claude-opus-4-8",
@@ -213,6 +232,20 @@ AZURE_AI_MODELS: Tuple[ModelEntry, ...] = (
 
 
 VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="vertex-claude-fable-5",
+        model="vertex_ai/claude-fable-5",
+        mode="adaptive",
+        extra_params=(("vertex_location", "global"),),
+        required_env=_VERTEX_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        fail_reason=(
+            "claude-fable-5 availability on the CI Vertex project is not yet "
+            "confirmed for this brand-new release, so this cell stays loud in "
+            "CI until verified. Remove this fail_reason once the model is "
+            "confirmed available on the global Vertex endpoint."
+        ),
+    ),
     ModelEntry(
         alias="vertex-claude-opus-4-8",
         model="vertex_ai/claude-opus-4-8",
@@ -263,6 +296,23 @@ VERTEX_AI_MODELS: Tuple[ModelEntry, ...] = (
 
 
 BEDROCK_CONVERSE_MODELS: Tuple[ModelEntry, ...] = (
+    ModelEntry(
+        alias="bedrock-claude-fable-5",
+        model="bedrock/converse/us.anthropic.claude-fable-5",
+        mode="adaptive",
+        extra_params=(("aws_region_name", "us-east-1"),),
+        required_env=_BEDROCK_REQ,
+        caps=_CAPS_XHIGH_MAX,
+        bedrock_effort_ceiling="xhigh",
+        unavailable_error="is not available for this account",
+        fail_reason=(
+            "claude-fable-5 on Bedrock requires the account to opt in to "
+            "provider data sharing (data retention mode "
+            "'provider_data_sharing' via the Data Retention API); the CI "
+            "account has not opted in yet, so this cell stays loud in CI. "
+            "Remove this fail_reason once the opt-in is done."
+        ),
+    ),
     ModelEntry(
         alias="bedrock-claude-opus-4-8",
         model="bedrock/converse/us.anthropic.claude-opus-4-8",

tests/llm_translation/reasoning_effort_grid/test_reasoning_effort_grid.py

@@ -15,7 +15,6 @@ from .grid_spec import (
     all_cells,
 )
 
-
 _PROMPT_MESSAGES: List[Dict[str, str]] = [
     {"role": "user", "content": "Step by step, calculate 47 * 53. Show your work."}
 ]
@@ -201,8 +200,8 @@ async def test_reasoning_effort_grid(
 
 
 def test_grid_cell_count() -> None:
-    assert len(_PARAMS) == 25 * 11, (
-        f"expected 275 cells (25 provider x model combos x 11 efforts), "
+    assert len(_PARAMS) == 29 * 11, (
+        f"expected 319 cells (29 provider x model combos x 11 efforts), "
         f"got {len(_PARAMS)}"
     )
 

tests/llm_translation/test_llm_response_utils/test_convert_dict_to_chat_completion.py

@@ -2425,6 +2425,42 @@ class TestConvertToModelResponseObjectCompletion:
         assert result.choices[0].message.content == "The answer is 4."
         assert result.choices[0].message.reasoning_content == "2+2=4"
 
+    def test_reasoning_content_not_mirrored_into_provider_specific_fields(self):
+        """Mirroring reasoning_content into provider_specific_fields made
+        cache-replayed messages diverge from live Anthropic messages, which
+        only set it top-level, breaking cache key stability (issue #27337)."""
+        response_object = {
+            "id": "chatcmpl-5",
+            "model": "claude-sonnet-4-5",
+            "choices": [
+                {
+                    "finish_reason": "stop",
+                    "index": 0,
+                    "message": {
+                        "content": "The answer is 4.",
+                        "role": "assistant",
+                        "reasoning_content": "2+2=4",
+                        "thinking_blocks": [
+                            {
+                                "type": "thinking",
+                                "thinking": "2+2=4",
+                                "signature": "sig",
+                            }
+                        ],
+                    },
+                }
+            ],
+            "usage": {"prompt_tokens": 5, "completion_tokens": 10, "total_tokens": 15},
+        }
+
+        result = convert_to_model_response_object(
+            response_object=response_object,
+            model_response_object=ModelResponse(),
+        )
+        message = result.choices[0].message
+        assert message.reasoning_content == "2+2=4"
+        assert "reasoning_content" not in (message.provider_specific_fields or {})
+
     def test_response_none_raises(self):
         with pytest.raises(Exception):
             convert_to_model_response_object(

tests/local_testing/test_basic_python_version.py

@@ -92,6 +92,56 @@ def test_package_dependencies():
         )
 
 
+def test_cli_extra_is_a_thin_client_install():
+    """The `cli` extra must install a working `lite` client without dragging in the
+    proxy server runtime. It therefore has to declare the CLI's real third-party
+    deps (rich, pyyaml, requests) and must never contain a server-only dependency
+    from the `proxy` extra; a leak there silently re-bloats the laptop install.
+    """
+    import pathlib
+
+    import litellm
+    from packaging.requirements import Requirement
+
+    try:
+        import tomllib as tomli
+    except ImportError:
+        try:
+            import tomli
+        except ImportError:
+            pytest.skip("tomli/tomllib not available - skipping dependency check")
+
+    pyproject_path = pathlib.Path(litellm.__file__).parent.parent / "pyproject.toml"
+    with open(pyproject_path, "rb") as f:
+        optional_deps = tomli.load(f)["project"]["optional-dependencies"]
+
+    assert "cli" in optional_deps, "Expected a `cli` extra for the thin lite install"
+
+    cli_names = {Requirement(req).name.lower() for req in optional_deps["cli"]}
+
+    missing = {"rich", "pyyaml", "requests"} - cli_names
+    assert not missing, f"`cli` extra is missing deps the lite CLI imports: {missing}"
+
+    server_only = {
+        "fastapi",
+        "uvicorn",
+        "gunicorn",
+        "granian",
+        "starlette",
+        "boto3",
+        "polars",
+        "soundfile",
+        "mcp",
+        "cryptography",
+        "apscheduler",
+        "rq",
+        "litellm-enterprise",
+        "litellm-proxy-extras",
+    }
+    leaked = cli_names & server_only
+    assert not leaked, f"`cli` extra leaks proxy-server deps onto laptops: {leaked}"
+
+
 import os
 import subprocess
 import time

tests/local_testing/test_router_debug_logs.py

@@ -82,7 +82,9 @@ def test_async_fallbacks(caplog):
     asyncio.run(_make_request())
     captured_logs = [rec.message for rec in caplog.records]
 
-    # on circle ci the captured logs get some async task exception logs - filter them out "Task exception was never retrieved"
+    # on circle ci the captured logs get async cleanup noise from the gc (leaked
+    # task warnings, plus aiohttp "Unclosed client session"/"Unclosed connector"
+    # warnings from cached clients other router tests evicted) - filter it out
     captured_logs = [
         log
         for log in captured_logs
@@ -90,6 +92,8 @@ def test_async_fallbacks(caplog):
         and "Task was destroyed but it is pending" not in log
         and "get_available_deployment" not in log
         and "in the Langfuse queue" not in log
+        and "Unclosed client session" not in log
+        and "Unclosed connector" not in log
     ]
 
     print("\n Captured caplog records - ", captured_logs)