Merge remote-tracking branch 'origin/main' into litellm_fips_b0_boot_gate

litellm/_lazy_imports_registry.py

@@ -1469,11 +1469,11 @@ _UTILS_MODULE_IMPORT_MAP: Final = {
     "AsyncHTTPHandler": ("litellm.llms.custom_httpx.http_handler", "AsyncHTTPHandler"),
     "HTTPHandler": ("litellm.llms.custom_httpx.http_handler", "HTTPHandler"),
     "get_num_retries_from_retry_policy": (
-        "litellm.router_utils.get_retry_from_policy",
+        "litellm.litellm_core_utils.get_retry_from_policy",
         "get_num_retries_from_retry_policy",
     ),
     "reset_retry_policy": (
-        "litellm.router_utils.get_retry_from_policy",
+        "litellm.litellm_core_utils.get_retry_from_policy",
         "reset_retry_policy",
     ),
     "get_coroutine_checker": (

litellm/a2a_protocol/litellm_completion_bridge/handler.py

@@ -22,7 +22,7 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import (
     A2AStreamingContext,
 )
 from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager
-from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS
+from litellm.constants import A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS, A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
 from litellm.interactions.agents.utils import merge_agent_headers
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.types.utils import ModelResponse
@@ -268,7 +268,7 @@ class A2ACompletionBridgeHandler:
             ctx=ctx,
             state="working",
             final=False,
-            message_text="Processing request...",
+            message_text=A2A_COMPLETION_BRIDGE_WORKING_MESSAGE,
         )
         yield working_event
 

litellm/a2a_protocol/main.py

@@ -191,6 +191,23 @@ def _set_litellm_params_on_logging_obj(
     }
 
 
+async def _record_send_message_billing(
+    kwargs: Mapping[str, object],
+    request: "SendMessageRequest",
+    response_dict: Mapping[str, object],
+    litellm_params: Mapping[str, object],
+    agent_id: str | None,
+) -> None:
+    prompt_tokens, completion_tokens, _ = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
+        request=request,
+        response_dict=response_dict,
+    )
+    _set_usage_on_logging_obj(kwargs=kwargs, prompt_tokens=prompt_tokens, completion_tokens=completion_tokens)
+    if "error" not in response_dict:
+        _set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
+    _set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
+
+
 def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: Mapping[str, object]) -> str:
     """
     Extract agent info and set model/custom_llm_provider for cost tracking.
@@ -483,13 +500,21 @@ async def asend_message(
     if custom_llm_provider:
         if request is None:
             raise ValueError("request is required for completion bridge")
-        return await _send_message_via_completion_bridge(
+        bridge_response: Final = await _send_message_via_completion_bridge(
             request=request,
             custom_llm_provider=custom_llm_provider,
             api_base=api_base,
             litellm_params=litellm_params,
             agent_extra_headers=agent_extra_headers,
         )
+        await _record_send_message_billing(
+            kwargs=kwargs,
+            request=request,
+            response_dict=bridge_response.model_dump(mode="json", exclude_none=True),
+            litellm_params=litellm_params,
+            agent_id=agent_id,
+        )
+        return bridge_response
 
     # Standard A2A client flow
     if request is None:
@@ -538,30 +563,14 @@ async def asend_message(
     # Wrap in LiteLLM response type for _hidden_params support
     response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id))
 
-    # Calculate token usage from request and response
-    response_dict: Final[dict[str, object]] = a2a_response.root.model_dump(mode="json", exclude_none=True)
-    (
-        prompt_tokens,
-        completion_tokens,
-        _,
-    ) = await asyncify(A2ARequestUtils.calculate_usage_from_request_response)(
-        request=request,
-        response_dict=response_dict,
-    )
-
-    # Set usage on logging obj for standard logging payload
-    _set_usage_on_logging_obj(
+    await _record_send_message_billing(
         kwargs=kwargs,
-        prompt_tokens=prompt_tokens,
-        completion_tokens=completion_tokens,
+        request=request,
+        response_dict=a2a_response.root.model_dump(mode="json", exclude_none=True),
+        litellm_params=litellm_params,
+        agent_id=agent_id,
     )
 
-    # Merge agent pricing params into the logging obj so cost is calculated
-    _set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
-
-    # Set agent_id on logging obj for SpendLogs tracking
-    _set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
-
     return response
 
 
@@ -646,6 +655,7 @@ async def asend_message_streaming(
     metadata: dict[str, object] | None = None,
     proxy_server_request: dict[str, object] | None = None,
     agent_extra_headers: dict[str, str] | None = None,
+    agent_name: str | None = None,
     **kwargs: object,
 ) -> AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"]:
     """
@@ -661,6 +671,7 @@ async def asend_message_streaming(
         agent_id: Optional agent ID for tracking in SpendLogs
         metadata: Optional metadata dict (contains user_api_key, user_id, team_id, etc.)
         proxy_server_request: Optional proxy server request data
+        agent_name: Agent name logged as a2a_agent/<agent_name> for completion bridge agents
 
     Yields:
         SendStreamingMessageResponse chunks from the agent
@@ -685,6 +696,7 @@ async def asend_message_streaming(
             print(chunk)
         ```
     """
+    called_at: Final = datetime.datetime.now()
     litellm_params = litellm_params or {}
     custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
 
@@ -705,20 +717,47 @@ async def asend_message_streaming(
             request.params.model_dump(mode="json") if hasattr(request.params, "model_dump") else dict(request.params)
         )
 
-        async for chunk in A2ACompletionBridgeHandler.handle_streaming(
+        bridge_stream: Final = A2ACompletionBridgeHandler.handle_streaming(
             request_id=str(request.id),
             params=params,
             litellm_params=litellm_params,
             api_base=api_base,
             agent_extra_headers=agent_extra_headers,
-        ):
-            yield chunk
+        )
+        caller_logging_obj: Final = kwargs.get("litellm_logging_obj")
+        bridge_logging_obj: Final = (
+            caller_logging_obj
+            if isinstance(caller_logging_obj, Logging)
+            else _build_streaming_logging_obj(
+                request=request,
+                agent_name=agent_name or "unknown",
+                agent_id=agent_id,
+                litellm_params=litellm_params,
+                metadata=metadata,
+                proxy_server_request=proxy_server_request,
+                start_time=called_at,
+            )
+        )
+        _set_litellm_params_on_logging_obj(kwargs=kwargs, litellm_params=litellm_params)
+        _set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
+
+        bridge_iterator: Final = A2AStreamingIterator(
+            stream=bridge_stream,
+            request=request,
+            logging_obj=bridge_logging_obj,
+            agent_name=agent_name or "unknown",
+        )
+        try:
+            async for chunk in bridge_iterator:
+                yield chunk
+        except (GeneratorExit, asyncio.CancelledError):
+            await bridge_iterator.abill_on_disconnect()
+            raise
         return
 
     if request is None:
         raise ValueError("request is required")
 
-    called_at: Final = datetime.datetime.now()
     _raw_logging_obj: Final = kwargs.get("litellm_logging_obj")
     logging_obj: Logging | None = _raw_logging_obj if isinstance(_raw_logging_obj, Logging) else None
 
@@ -742,12 +781,12 @@ async def asend_message_streaming(
 
     assert a2a_client is not None
 
-    agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
+    card_agent_name: Final = _get_a2a_model_info(a2a_client, kwargs)
 
     if logging_obj is None:
         logging_obj = _build_streaming_logging_obj(
             request=request,
-            agent_name=agent_name,
+            agent_name=card_agent_name,
             agent_id=agent_id,
             litellm_params=litellm_params,
             metadata=metadata,
@@ -755,7 +794,7 @@ async def asend_message_streaming(
             start_time=called_at,
         )
 
-    verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, agent_name)
+    verbose_logger.info("A2A send_message_streaming request_id=%s, agent=%s", request.id, card_agent_name)
 
     agent_card: Final = _get_a2a_client_agent_card(a2a_client)
     card_url: Final = get_agent_card_url(agent_card) if agent_card else None
@@ -766,18 +805,23 @@ async def asend_message_streaming(
         agent_card=agent_card,
         card_url=card_url,
         api_base=api_base,
-        agent_name=agent_name,
+        agent_name=card_agent_name,
     )
 
     _set_agent_id_on_logging_obj(kwargs=kwargs, agent_id=agent_id)
 
-    async for chunk in A2AStreamingIterator(
+    url_agent_iterator: Final = A2AStreamingIterator(
         stream=stream,
         request=request,
         logging_obj=logging_obj,
-        agent_name=agent_name,
-    ):
-        yield chunk
+        agent_name=card_agent_name,
+    )
+    try:
+        async for chunk in url_agent_iterator:
+            yield chunk
+    except (GeneratorExit, asyncio.CancelledError):
+        await url_agent_iterator.abill_on_disconnect()
+        raise
 
 
 async def create_a2a_client(

litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py

@@ -515,6 +515,8 @@ class PydanticAITransformation:
                             "contextId": context_id,
                             "kind": "artifact-update",
                             "taskId": task_id,
+                            "append": i > 0,
+                            "lastChunk": is_last_chunk,
                             "artifact": {
                                 "artifactId": artifact_id,
                                 "parts": [

litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py

@@ -206,6 +206,8 @@ class WatsonxOrchestrateTransformation:
                     "contextId": context_id,
                     "kind": "artifact-update",
                     "taskId": task_id,
+                    "append": i > 0,
+                    "lastChunk": is_last,
                     "artifact": {
                         "artifactId": artifact_id,
                         "parts": [{"kind": "text", "text": chunk_text}],

litellm/a2a_protocol/streaming_iterator.py

@@ -3,16 +3,18 @@ A2A Streaming Iterator with token tracking and logging support.
 """
 
 import asyncio
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from datetime import datetime
 from typing import TYPE_CHECKING, Final
 
+import anyio
 from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
 from litellm.a2a_protocol.utils import A2ARequestUtils
+from litellm.constants import A2A_COMPLETION_BRIDGE_WORKING_MESSAGE
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -23,6 +25,23 @@ if TYPE_CHECKING:
 _OPAQUE_PAYLOAD: Final = TypeAdapter(object)
 
 
+def _chunk_payload(chunk: "SendStreamingMessageResponse | Mapping[str, object]") -> Mapping[str, object]:
+    if isinstance(chunk, Mapping):
+        return chunk
+    return chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
+
+
+def _artifact_update(payload: Mapping[str, object]) -> tuple[str, bool] | None:
+    result: Final = as_str_mapping(payload.get("result"))
+    if result is None or result.get("kind") != "artifact-update":
+        return None
+    artifact: Final = as_str_mapping(result.get("artifact"))
+    artifact_id: Final = artifact.get("artifactId") if artifact is not None else None
+    if not isinstance(artifact_id, str):
+        return None
+    return artifact_id, result.get("append") is True
+
+
 class A2AStreamingIterator:
     """
     Async iterator for A2A streaming responses with token tracking.
@@ -32,7 +51,7 @@ class A2AStreamingIterator:
 
     def __init__(
         self,
-        stream: AsyncIterator["SendStreamingMessageResponse"],
+        stream: AsyncIterator["SendStreamingMessageResponse | Mapping[str, object]"],
         request: "SendStreamingMessageRequest",
         logging_obj: LiteLLMLoggingObj,
         agent_name: str = "unknown",
@@ -44,14 +63,16 @@ class A2AStreamingIterator:
         self.start_time = logging_obj.start_time
 
         # Collect chunks for token counting
-        self.chunks: list[SendStreamingMessageResponse] = []
-        self.collected_text_parts: list[str] = []
-        self.final_chunk: SendStreamingMessageResponse | None = None
+        self.chunks: list[SendStreamingMessageResponse | Mapping[str, object]] = []
+        self.latest_message_text = ""
+        self.artifact_texts: Mapping[str, str] = {}
+        self.saw_error = False
+        self.final_chunk: SendStreamingMessageResponse | Mapping[str, object] | None = None
 
     def __aiter__(self):
         return self
 
-    async def __anext__(self) -> "SendStreamingMessageResponse":
+    async def __anext__(self) -> "SendStreamingMessageResponse | Mapping[str, object]":
         try:
             chunk: Final = await self.stream.__anext__()
             if self.logging_obj.completion_start_time is None:
@@ -59,38 +80,54 @@ class A2AStreamingIterator:
 
             # Store chunk
             self.chunks.append(chunk)
+            payload: Final = _chunk_payload(chunk)
+            self.saw_error = self.saw_error or payload.get("error") is not None
 
             # Extract text from chunk for token counting
-            self._collect_text_from_chunk(chunk)
+            self._collect_text_from_chunk(payload)
 
             # Check if this is the final chunk (completed status)
-            if self._is_completed_chunk(chunk):
+            if self._is_completed_chunk(payload):
                 self.final_chunk = chunk
 
             return chunk
 
         except StopAsyncIteration:
-            # Stream ended - handle logging
-            if self.final_chunk is None and self.chunks:
-                self.final_chunk = self.chunks[-1]
             await self._handle_stream_complete()
             raise
 
-    def _collect_text_from_chunk(self, chunk: "SendStreamingMessageResponse") -> None:
-        """Extract text from a streaming chunk and add to collected parts."""
+    async def abill_on_disconnect(self) -> None:
+        if not self.chunks:
+            return
+        with anyio.CancelScope(shield=True):
+            await self._handle_stream_complete()
+
+    def _collect_text_from_chunk(self, payload: Mapping[str, object]) -> None:
+        """Record the reply text a streaming chunk carries."""
         try:
-            chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
-            text: Final = A2ARequestUtils.extract_text_from_response(chunk_dict)
-            if text:
-                self.collected_text_parts.append(text)
+            text: Final = A2ARequestUtils.extract_text_from_stream_event(payload)
+            if not text:
+                return
+            artifact_update: Final = _artifact_update(payload)
+            if artifact_update is None and text == A2A_COMPLETION_BRIDGE_WORKING_MESSAGE:
+                return
+            if artifact_update is None:
+                self.latest_message_text = text
+                return
+            artifact_id, append = artifact_update
+            artifact_text: Final = self.artifact_texts.get(artifact_id, "") + text if append else text
+            self.artifact_texts = {**self.artifact_texts, artifact_id: artifact_text}
         except Exception:
             verbose_logger.debug("Failed to extract text from A2A streaming chunk")
 
-    def _is_completed_chunk(self, chunk: "SendStreamingMessageResponse") -> bool:
+    def _reply_text(self) -> str:
+        artifact_text: Final = " ".join(text for text in self.artifact_texts.values() if text)
+        return artifact_text or self.latest_message_text
+
+    def _is_completed_chunk(self, payload: Mapping[str, object]) -> bool:
         """Check if chunk indicates stream completion."""
         try:
-            chunk_dict: Final = chunk.model_dump(mode="json", exclude_none=True) if hasattr(chunk, "model_dump") else {}
-            result: Final = chunk_dict.get("result", {})
+            result: Final = payload.get("result", {})
             if isinstance(result, dict):
                 status: Final = result.get("status", {})
                 if isinstance(status, dict):
@@ -101,6 +138,8 @@ class A2AStreamingIterator:
 
     async def _handle_stream_complete(self) -> None:
         """Handle logging and token counting when stream completes."""
+        if self.final_chunk is None and self.chunks:
+            self.final_chunk = self.chunks[-1]
         try:
             end_time: Final = datetime.now()
 
@@ -109,9 +148,7 @@ class A2AStreamingIterator:
             input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
             prompt_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(input_text)
 
-            # Use the last (most complete) text from chunks
-            output_text: Final = self.collected_text_parts[-1] if self.collected_text_parts else ""
-            completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(output_text)
+            completion_tokens: Final = await asyncify(A2ARequestUtils.count_tokens)(self._reply_text())
 
             total_tokens: Final = prompt_tokens + completion_tokens
 
@@ -128,7 +165,7 @@ class A2AStreamingIterator:
             self.logging_obj.model_call_details["stream"] = False
 
             # Calculate cost using A2ACostCalculator
-            response_cost: Final = A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
+            response_cost: Final = 0.0 if self.saw_error else A2ACostCalculator.calculate_a2a_cost(self.logging_obj)
             self.logging_obj.model_call_details["response_cost"] = response_cost
 
             # Build result for logging
@@ -167,9 +204,7 @@ class A2AStreamingIterator:
         # Add final chunk result if available
         if self.final_chunk:
             try:
-                chunk_dict: Final = as_str_mapping(
-                    _OPAQUE_PAYLOAD.validate_python(self.final_chunk.model_dump(mode="json", exclude_none=True))
-                )
+                chunk_dict: Final = as_str_mapping(_OPAQUE_PAYLOAD.validate_python(_chunk_payload(self.final_chunk)))
                 if chunk_dict is not None:
                     result["result"] = chunk_dict.get("result", {})
             except Exception:

litellm/a2a_protocol/utils.py

@@ -6,13 +6,17 @@ import hashlib
 from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final, Protocol
 
+from pydantic import TypeAdapter
+
 import litellm
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.core_helpers import has_attributes
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, has_attributes
 
 if TYPE_CHECKING:
     from a2a.types import SendMessageRequest, SendStreamingMessageRequest
 
+_OBJECT_ITEMS: Final = TypeAdapter(tuple[object, ...])
+
 
 class _SupportsModelDump(Protocol):
     def model_dump(self, *, mode: str) -> object: ...
@@ -34,6 +38,9 @@ class A2ARequestUtils:
         """
         if message is None:
             return ""
+        # a2a-sdk Part RootModels keep kind/text on part.root, so read pydantic messages as dicts
+        if has_attributes(message, _SupportsModelDump, "model_dump"):
+            return A2ARequestUtils.extract_text_from_message(message.model_dump(mode="json"))
 
         # Handle both dict and object access
         if isinstance(message, dict):
@@ -63,8 +70,8 @@ class A2ARequestUtils:
         Returns:
             Text from response message parts
         """
-        result: Final = response_dict.get("result", {})
-        if not isinstance(result, dict):
+        result: Final = as_str_mapping(response_dict.get("result"))
+        if result is None:
             return ""
 
         # Direct message format (A2A spec): detect by explicit kind tag only.
@@ -72,10 +79,36 @@ class A2ARequestUtils:
         # type that happens to include a "parts" field.
         if result.get("kind") == "message":
             return A2ARequestUtils.extract_text_from_message(result)
+        if result.get("kind") == "task":
+            return A2ARequestUtils.extract_text_from_task(result)
 
         message: Final = result.get("message", {})
         return A2ARequestUtils.extract_text_from_message(message)
 
+    @staticmethod
+    def extract_text_from_task(task: Mapping[str, object]) -> str:
+        artifacts: Final = task.get("artifacts")
+        artifact_items: Final = _OBJECT_ITEMS.validate_python(artifacts) if isinstance(artifacts, list) else ()
+        artifact_texts: Final = tuple(
+            A2ARequestUtils.extract_text_from_message(artifact) for artifact in artifact_items
+        )
+        if any(artifact_texts):
+            return " ".join(text for text in artifact_texts if text)
+        status: Final = as_str_mapping(task.get("status"))
+        return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
+
+    @staticmethod
+    def extract_text_from_stream_event(event: Mapping[str, object]) -> str:
+        result: Final = as_str_mapping(event.get("result"))
+        if result is None:
+            return ""
+        if result.get("kind") == "artifact-update":
+            return A2ARequestUtils.extract_text_from_message(result.get("artifact"))
+        if result.get("kind") == "status-update":
+            status: Final = as_str_mapping(result.get("status"))
+            return A2ARequestUtils.extract_text_from_message(status.get("message")) if status is not None else ""
+        return A2ARequestUtils.extract_text_from_response(event)
+
     @staticmethod
     def get_input_message_from_request(
         request: "SendMessageRequest | SendStreamingMessageRequest",
@@ -128,17 +161,8 @@ class A2ARequestUtils:
         Returns:
             Tuple of (prompt_tokens, completion_tokens, total_tokens)
         """
-        # Count input tokens. Dump the message to a dict first so extraction hits
-        # the dict branch — request-side parts are a2a-sdk Part RootModels whose
-        # kind/text live on part.root, which the object branch cannot read. This
-        # mirrors how the response side already works (it operates on model_dump).
         input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
-        input_text: Final = A2ARequestUtils.extract_text_from_message(
-            input_message.model_dump(mode="json")
-            if has_attributes(input_message, _SupportsModelDump, "model_dump")
-            else input_message
-        )
-        prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
+        prompt_tokens: Final = A2ARequestUtils.count_tokens(A2ARequestUtils.extract_text_from_message(input_message))
 
         # Count output tokens
         output_text: Final = A2ARequestUtils.extract_text_from_response(response_dict)

litellm/constants.py

@@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
 # Anthropic's Messages API rejects thinking.budget_tokens < 1024.
 ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
+ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
+ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
 ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
 REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
 DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
@@ -902,6 +904,7 @@ PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO: Final = frozenset(
 
 SUBSCRIPTION_BACKED_PROVIDERS: Final = frozenset({"chatgpt"})
 A2A_CHAT_COMPLETION_BRIDGE_PROVIDERS: Final = frozenset({"databricks_agent"})
+A2A_COMPLETION_BRIDGE_WORKING_MESSAGE: Final = "Processing request..."
 
 LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS: Final = [
     "openai",
@@ -1162,6 +1165,7 @@ openai_compatible_providers: Final[list[str]] = [
     "cortecs",
     "scx-ai",
     "prism",
+    "hostinger_router",
     "sail",
     "reka",
     "streamlake",
@@ -2046,6 +2050,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
 SPECIAL_LITELLM_AUTH_TOKEN: Final = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = get_env_int("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60)
 AUTH_OBJECTS_TARGET: Final = "auth_objects"
+HEALTH_CHECKS_TARGET: Final = "health_checks"
 DEFAULT_ACCESS_GROUP_CACHE_TTL: Final = get_env_int("DEFAULT_ACCESS_GROUP_CACHE_TTL", 600)
 SPEND_LOG_KEY_METADATA_CACHE_TTL: Final = 600
 SPEND_LOG_KEY_METADATA_MISS_CACHE_TTL: Final = 30

litellm/integrations/anthropic_cache_control_hook.py

@@ -14,7 +14,7 @@ import itertools
 import os
 import re
 from collections.abc import Callable, Container, Iterable, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
 from pydantic import TypeAdapter, ValidationError
@@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
         return None
 
 
+_MessageT: Final = TypeVar("_MessageT")
+_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
+_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
+
+
+def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
+    lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
+    return tuple(
+        itertools.chain.from_iterable(
+            _OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
+        )
+    )
+
+
+def _block_carries_cache_control(block: object) -> bool:
+    return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
+
+
+_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
+
+
+def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
+    if fields.get("cache_control") is None:
+        return False
+    return (
+        not isinstance(fields.get("content"), list)
+        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+    )
+
+
+def cache_control_count(message: object) -> int:
+    if not isinstance(message, dict):
+        return 0
+    fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
+    block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
+    return int(_message_mark_reaches_provider(fields)) + block_marks
+
+
+def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
+    others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
+    if "cache_control" not in fields:
+        return others
+    cache_control: Final = rewrite(fields["cache_control"])
+    return others if cache_control is None else {**others, "cache_control": cache_control}
+
+
+def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
+    if not isinstance(block, dict):
+        return block
+    return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
+
+
+def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
+    if not isinstance(message, dict):
+        return message
+    fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
+    blocks_rewritten: Final = {
+        key: (
+            [block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
+            if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
+            else value
+        )
+        for key, value in fields.items()
+    }
+    return cast(  # cast-ok: the same message shape with each cache marker rewritten or dropped
+        _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
+    )
+
+
+def dropped_cache_control(_: object) -> None:
+    return None
+
+
+def _first_marked_index(value: object) -> int | None:
+    if not isinstance(value, list):
+        return None
+    blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
+    return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
+
+
+_MarkSlot: TypeAlias = tuple[str, int]
+_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
+
+
+def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
+    value: Final = fields.get(key)
+    if not isinstance(value, list):
+        return ()
+    blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
+    return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
+
+
+def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
+    content_slots: Final = _block_mark_slots(fields, "content")
+    tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
+    if not _message_mark_reaches_provider(fields):
+        return content_slots + tool_call_slots
+    if isinstance(fields.get("content"), list):
+        return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
+    return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
+
+
+def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
+    if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
+        return value
+    return [
+        block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
+        for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
+    ]
+
+
+def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
+    if dropped == 0 or not isinstance(message, dict):
+        return message
+    fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
+    dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
+    blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
+    return cast(  # cast-ok: the same message with its oldest wire-order marks dropped
+        _MessageT,
+        _fields_with_cache_control(blocks_unmarked, dropped_cache_control)
+        if _MESSAGE_MARK_SLOT in dropped_slots
+        else blocks_unmarked,
+    )
+
+
+def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
+    if not isinstance(message, dict):
+        return message
+    fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
+    content: Final = fields.get("content")
+    if fields.get("role") != "tool" or not isinstance(content, list):
+        return message
+    kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
+    return cast(  # cast-ok: the same tool message keeping only the mark its single cache point reads
+        _MessageT,
+        {
+            **fields,
+            "content": [
+                block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
+                for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
+            ],
+        },
+    )
+
+
+_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
+
+
+def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
+    remaining, _ = state
+    kept: Final = min(marks, remaining)
+    return remaining - kept, marks - kept
+
+
+def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
+    remaining, _ = state
+    if marks > remaining:
+        return remaining, marks
+    return remaining - marks, 0
+
+
+def _messages_within_budget(
+    messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
+) -> Sequence[_MessageT]:
+    budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
+    newest_first: Final = tuple(reversed(messages))
+    budget_after_each: Final = itertools.accumulate(
+        (cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
+    )
+    dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
+    kept_newest_first: Final = tuple(
+        _without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
+    )
+    return tuple(reversed(kept_newest_first))
+
+
+def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
+    return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
+
+
+def whole_messages_within_cache_control_limit(
+    messages: Sequence[_MessageT], reserved_marks: int
+) -> Sequence[_MessageT]:
+    return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
+
+
 def _part_with_cache_control_renamed(part: object) -> object:
     if not _accepts_prompt_cache_breakpoint(part):
         return part
@@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 llm_router=None,
             )
         return None
+
+
+def messages_within_cache_control_cap(
+    messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
+) -> Sequence[_MessageT]:
+    """Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
+
+    Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
+    """
+    reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
+        _validated_object_list(tools), cache_control, request_kwargs
+    )
+    kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
+    dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
+    if dropped > 0:
+        verbose_logger.warning(
+            "Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
+            "provider accepts at most %d cache_control blocks per request",
+            dropped,
+            MAX_CACHE_CONTROL_BLOCKS,
+        )
+    return kept

litellm/integrations/prometheus.py

@@ -66,7 +66,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import EndUserRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
-from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
+from litellm.router_api import deployment_metadata
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
@@ -3022,8 +3022,8 @@ class PrometheusLogger(CustomLogger):
         ).get("custom_llm_provider")
         if isinstance(provider, str) and provider:
             return provider
-        deployment_metadata: Final = find_deployment_metadata(litellm_params)
-        return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
+        picked_deployment: Final = deployment_metadata(litellm_params)
+        return _deployment_model_provider(picked_deployment.get("deployment") if picked_deployment else None)
 
     @staticmethod
     def _extract_api_provider_from_exception(exception: Exception) -> str | None:
@@ -5209,7 +5209,7 @@ def _tag_matches_wildcard_configured_pattern(tags: Sequence[str], configured_tag
     """
     import re
 
-    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+    from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
 
     pattern_router: Final = PatternMatchRouter()
     regex_pattern: Final = pattern_router.pattern_to_regex(configured_tag)

litellm/litellm_core_utils/fallback_utils.py

@@ -3,13 +3,13 @@ from typing import Final
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.add_retry_fallback_headers import (
+    add_fallback_headers_to_response,
+)
 from litellm.litellm_core_utils.core_helpers import (
     filter_internal_params,
     safe_deep_copy,
 )
-from litellm.router_utils.add_retry_fallback_headers import (
-    add_fallback_headers_to_response,
-)
 
 from .asyncify import run_async_function
 

litellm/litellm_core_utils/health_check_helpers.py

@@ -119,7 +119,7 @@ def _is_route_segment(custom_llm_provider: str, segment: str, model_after_segmen
 
 def wildcard_health_check_models(wildcard: str, custom_llm_provider: str) -> tuple[str, ...]:
     from litellm.litellm_core_utils.llm_request_utils import rank_chat_models_from_llm_provider
-    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+    from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
 
     provider_prefix: Final = f"{custom_llm_provider}/"
     segments: Final = wildcard.removeprefix(provider_prefix).split("/")

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
             usage=usage, response_object=response_object
         )
 
+        response_model: Final = _output_item_field(response_object, "model")
         if model_info is not None and resolved_usage is not None and resolved_provider is not None:
             result: Final = get_cost_for_web_search_request(
                 custom_llm_provider=resolved_provider,
                 usage=resolved_usage,
                 model_info=model_info,
+                response_model=response_model if isinstance(response_model, str) else None,
+                lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
             )
             if result is not None:
                 return result

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
         return [{"input": str(messages)}]
 
 
+def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
+    texts: Final = convert_content_list_to_str(message=message)
+    if not texts:
+        return message
+    return cast(  # cast-ok: the same message with its content flattened to one string
+        AllMessageValues, {**message, "content": texts}
+    )
+
+
 def handle_messages_with_content_list_to_str_conversion(
     messages: list[AllMessageValues],
 ) -> list[AllMessageValues]:
     """
-    Handles messages with content list conversion
+    Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
     """
-    for message in messages:
-        texts = convert_content_list_to_str(message=message)
-        if texts:
-            message["content"] = texts
-    return messages
+    return [_message_with_str_content(message) for message in messages]
 
 
 def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

litellm/litellm_core_utils/ptu_shares.py

@@ -12,13 +12,13 @@ from datetime import date
 from typing import Final, Generic, TypeAlias, TypeVar
 
 from litellm.litellm_core_utils.ptu_pricing import parsed_ptu_shares, ptu_terms
+from litellm.litellm_core_utils.routing_common_utils import get_request_team_id, team_may_use_deployment
 from litellm.llms.azure.ptu_capacity import (
     PTUCapacity,
     deployment_ptu_capacity,
     is_azure_deployment,
     is_str_keyed_mapping,
 )
-from litellm.router_utils.common_utils import get_request_team_id, team_may_use_deployment
 
 _DeploymentT: Final = TypeVar("_DeploymentT", bound=Mapping[str, object])
 WildcardRoute: TypeAlias = Callable[[str], Sequence[Mapping[str, object]]]

litellm/litellm_core_utils/token_counter.py

@@ -56,9 +56,9 @@ from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEnc
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
+from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
 from litellm.litellm_core_utils.url_utils import safe_get
 from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
-from litellm.router_utils.deployment_pricing import deployment_pricing_models
 from litellm.rust_bridge.tokenizer import get_encoding
 from litellm.types.llms.anthropic import (
     AnthropicContentParamSource,

litellm/llms/__init__.py

@@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
 from . import *
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
     from litellm.llms.base_llm.guardrail_translation.base_translation import (
         BaseTranslation,
     )
@@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
     return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
 
 
-def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
+def _with_served_model_search_pricing(
+    model_info: "ModelInfo",
+    response_model: str | None,
+    served_model_provider: str,
+    lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
+) -> "ModelInfo":
+    if model_info.get("search_context_cost_per_query") is not None or response_model is None:
+        return model_info
+    served_model_info: Final = (
+        lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
+    )
+    if served_model_info is None:
+        return model_info
+    return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
+
+
+def get_cost_for_web_search_request(
+    custom_llm_provider: str,
+    usage: "Usage",
+    model_info: "ModelInfo",
+    response_model: str | None = None,
+    lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
+) -> float | None:
     """
     Get the cost for a web search request for a given model.
 
@@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
         custom_llm_provider: The custom LLM provider.
         usage: The usage object.
         model_info: The model info.
+        response_model: The model the provider response names.
+        lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
     """
     if custom_llm_provider == "gemini":
         from .gemini.cost_calculator import cost_per_web_search_request
@@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
     elif custom_llm_provider == "anthropic":
         from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
 
-        return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
+        return get_cost_for_anthropic_web_search(
+            model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
+            usage=usage,
+        )
     elif custom_llm_provider.startswith("vertex_ai"):
         # Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
         # (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
         # (which is the Gemini field). Route claude-* models to the Anthropic calculator.
         model_key: Final[str] = model_info.get("key", "") if model_info else ""
-        if "claude" in model_key.lower():
+        if "claude" in (response_model or model_key).lower():
             from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
 
             verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
-            return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
+            return get_cost_for_anthropic_web_search(
+                model_info=_with_served_model_search_pricing(
+                    model_info, response_model, "vertex_ai", lookup_model_info
+                ),
+                usage=usage,
+            )
 
         from .vertex_ai.gemini.cost_calculator import (
             cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
@@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
             return None
         from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
 
-        return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
+        return get_cost_for_anthropic_web_search(
+            model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
+            usage=usage,
+        )
     elif custom_llm_provider == "perplexity":
         # Perplexity handles search costs internally in its own cost calculator
         # Return 0.0 to indicate costs are already accounted for

litellm/llms/anthropic/chat/transformation.py

@@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
 _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _TOOLS_ADAPTER: Final = TypeAdapter(list[object])
 _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
     ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
 )
@@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             headers=headers,
         )
 
+    @staticmethod
+    def _messages_within_cache_control_cap(
+        messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
+    ) -> Sequence[AllMessageValues]:
+        from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
+
+        return messages_within_cache_control_cap(
+            messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
+        )
+
     def transform_request(
         self,
         model: str,
@@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             anthropic_messages_pt,
         )
 
-        drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
+        drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
+        capped_messages: Final = (
+            list(
+                self._messages_within_cache_control_cap(
+                    messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
+                )
+            )
+            if drop_params
+            else messages
+        )
 
-        if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
+        if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
             optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
 
         # Drop thinking param if thinking is enabled but thinking_blocks are missing
@@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         # Related issue: https://github.com/BerriAI/litellm/issues/18926
         if (
             optional_params.get("thinking") is not None
-            and messages is not None
-            and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
-            and not any_assistant_message_has_thinking_blocks(messages)
+            and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
+            and not any_assistant_message_has_thinking_blocks(capped_messages)
         ):
             if litellm.modify_params:
                 optional_params.pop("thinking", None)
@@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
             optional_params=optional_params,
         )
-        if _name_forward_map:
-            messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
+        request_messages: Final = (
+            self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
+            if _name_forward_map
+            else capped_messages
+        )
         if _name_reverse_map and isinstance(litellm_params, dict):
             litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
 
         # Only the leading system run becomes the top-level system prompt. A later
         # system message stays in the conversation: hoisting it rewrites the cached
         # prefix and re-bills the whole history at cache-write pricing (#36559).
-        leading_system_run, later_messages = split_leading_system_run(messages)
+        leading_system_run, later_messages = split_leading_system_run(request_messages)
         anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
         # Handling anthropic API Prompt Caching
         if len(anthropic_system_message_list) > 0:
@@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         except Exception as e:
             raise AnthropicError(
                 status_code=400,
-                message=f"{e}\nReceived Messages={messages}",
+                message=f"{e}\nReceived Messages={request_messages}",
             )  # don't use verbose_logger.exception, if exception is raised
 
         self.update_headers_with_optional_anthropic_beta(

litellm/llms/anthropic/common_utils.py

@@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
 
 import copy
 import json
+import os
 import re
 from collections.abc import Container, Mapping, MutableMapping, Sequence
 from dataclasses import dataclass
@@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
+    ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
+    ANTHROPIC_MAX_USER_ID_LENGTH,
     DEFAULT_MODEL_CREATED_AT_TIME,
     DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
     return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
 
 
+def anthropic_pass_through_api_base() -> str:
+    return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
+
+
+def _same_host_and_port(url: httpx.URL, other: str) -> bool:
+    try:
+        other_url: Final = httpx.URL(other)
+    except httpx.InvalidURL:
+        return False
+    return (url.host, url.port) == (other_url.host, other_url.port)
+
+
+def anthropic_upstream_user_id_limit(url: str) -> int | None:
+    upstream: Final = httpx.URL(url)
+    if not upstream.path.removesuffix("/").endswith("/v1/messages"):
+        return None
+    if upstream.host == "api.anthropic.com":
+        return ANTHROPIC_MAX_USER_ID_LENGTH
+    return (
+        ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
+        if _same_host_and_port(upstream, anthropic_pass_through_api_base())
+        else None
+    )
+
+
+def _has_email_shape(token: str) -> bool:
+    at_sign: Final = token.find("@", 1)
+    return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
+
+
+def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
+    return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
+
+
 def is_anthropic_messages_url(url: str) -> bool:
     """Check whether a URL addresses Anthropic's Messages API."""
     parsed_url: Final = urlparse(url)

litellm/llms/anthropic/messages/native/handler.py

@@ -19,6 +19,13 @@ from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheCont
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.reasoning_effort_capability import (
+    RequestedReasoningEffort,
+    deployment_reasoning_effort_levels,
+    keeps_effort,
+    model_group_from_metadata,
+    resolve_deployment_reasoning_effort,
+)
 from litellm.llms.anthropic.common_utils import (
     AnthropicModelInfo,
     strip_provider_specific_fields_from_anthropic_messages,
@@ -31,13 +38,6 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
-from litellm.router_utils.reasoning_effort_capability import (
-    RequestedReasoningEffort,
-    deployment_reasoning_effort_levels,
-    keeps_effort,
-    model_group_from_metadata,
-    resolve_deployment_reasoning_effort,
-)
 from litellm.types.llms.anthropic import AllAnthropicToolsValues
 from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
 from litellm.types.llms.anthropic_messages.anthropic_response import (

litellm/llms/anthropic/messages/native/interceptors/advisor.py

@@ -22,7 +22,7 @@ import litellm
 import litellm.constants as _c
 from litellm.litellm_core_utils.url_utils import validate_url
 from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages
-from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure
+from litellm.router_api import mark_advisor_orchestration_failure
 from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,

litellm/llms/anthropic/messages/utils.py

@@ -214,7 +214,7 @@ def normalize_reasoning_effort_value(
     if chain is None:
         return effort
 
-    from litellm.router_utils.reasoning_effort_capability import (
+    from litellm.litellm_core_utils.reasoning_effort_capability import (
         deployment_rounds_reasoning_effort,
         resolve_supported_reasoning_efforts,
     )

litellm/llms/bedrock/chat/converse_transformation.py

@@ -25,6 +25,7 @@ from litellm.constants import (
 )
 from litellm.litellm_core_utils.core_helpers import (
     as_str_mapping,
+    drops_unsupported_params,
     filter_exceptions_from_params,
     filter_internal_params,
     map_finish_reason,
@@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
 
         return data
 
+    @staticmethod
+    def _claude_messages_within_cache_control_cap(
+        messages: Sequence[AllMessageValues],
+        capability_model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> Sequence[AllMessageValues]:
+        if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
+            return messages
+        from litellm.integrations.anthropic_cache_control_hook import (
+            messages_within_cache_control_cap,
+            tool_message_with_single_cache_mark,
+        )
+
+        return messages_within_cache_control_cap(
+            tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
+        )
+
     async def _async_transform_request(
         self,
         model: str,
@@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
         base_model: Final = litellm_params.get("base_model")
         capability_model: Final = bedrock_capability_model(model, base_model)
         messages, system_content_blocks = self._transform_system_message(
-            messages, model=capability_model, deployment_model=model
+            list(
+                self._claude_messages_within_cache_control_cap(
+                    messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
+                )
+            ),
+            model=capability_model,
+            deployment_model=model,
         )
 
         # Convert last user message to guarded_text if guardrailConfig is present
@@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
         base_model: Final = litellm_params.get("base_model")
         capability_model: Final = bedrock_capability_model(model, base_model)
         messages, system_content_blocks = self._transform_system_message(
-            messages, model=capability_model, deployment_model=model
+            list(
+                self._claude_messages_within_cache_control_cap(
+                    messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
+                )
+            ),
+            model=capability_model,
+            deployment_model=model,
         )
 
         # Convert last user message to guarded_text if guardrailConfig is present

litellm/llms/clf_ai_gateway/chat/transformation.py

@@ -4,8 +4,8 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
-from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.secret_managers.main import get_secret_str
 
 SUPPORTED_OPENAI_PARAMS: Final = (

litellm/llms/compaction.py

@@ -9,7 +9,7 @@ from pydantic import TypeAdapter
 from litellm.types.llms.openai import AllMessageValues
 
 if TYPE_CHECKING:
-    from litellm.router import Router
+    from litellm.router_api import RoutingContext
 
 CompactionProtocol: TypeAlias = Literal["chat", "messages"]
 _MAPPING: Final = TypeAdapter(Mapping[str, object])
@@ -32,7 +32,9 @@ def get_native_compaction_provider(params: Mapping[str, object]) -> NativeCompac
     return compaction if compaction.supports_native_compaction(params) else None
 
 
-async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mapping[str, object]) -> Mapping[str, object]:
+async def dispatch(
+    router: RoutingContext, protocol: CompactionProtocol, payload: Mapping[str, object]
+) -> Mapping[str, object]:
     if protocol == "messages":
         return _MAPPING.validate_python(
             await router.aanthropic_messages(custom_llm_provider=None, client=None, **payload)
@@ -45,4 +47,4 @@ async def dispatch(router: Router, protocol: CompactionProtocol, payload: Mappin
             {key: value for key, value in payload.items() if key not in ("model", "messages", "stream")}
         ),
     )
-    return _MAPPING.validate_python(response.model_dump())
+    return _MAPPING.validate_python(response.model_dump())  # pyright: ignore[reportAttributeAccessIssue]  # stream=False: acompletion returns a ModelResponse

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -110,6 +110,15 @@ def map_aiohttp_exceptions() -> Generator[None, None, None]:
         raise mapped_exc(message) from exc
 
 
+def _unread_payload(content: object) -> bytes:
+    """Bytes the upstream sent before the connection dropped that aiohttp buffered but never handed out, since its
+    reads raise the stored payload error ahead of the buffer."""
+    if not isinstance(content, aiohttp.StreamReader):
+        return b""
+    buffered: Final = b"".join(content._buffer)  # pyright: ignore[reportPrivateUsage]  # no public read drains it
+    return buffered[content._buffer_offset :]  # pyright: ignore[reportPrivateUsage]  # see above
+
+
 def _release_transport_anchor(_transport: object) -> None:
     """Release a transport anchored to a response stream. See ``_anchor_transport_to``."""
 
@@ -144,6 +153,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
             raise httpx.ReadError(str(e)) from e
         except aiohttp.http_exceptions.TransferEncodingError as e:
             raise httpx.ReadError(str(e)) from e
+        except aiohttp.ClientPayloadError:
+            unread: Final = _unread_payload(self._aiohttp_response.content)
+            if unread:
+                yield unread
+            with map_aiohttp_exceptions():
+                raise
         except Exception:
             # For other exceptions, use the normal mapping
             with map_aiohttp_exceptions():

litellm/llms/databricks/embed/handler.py

@@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
         aembedding=None,
         custom_endpoint: bool | None = None,
         headers: dict | None = None,
+        drop_params: bool = False,
     ) -> EmbeddingResponse:
         # Check for custom user agent in optional_params or environment
         # This allows partners building on LiteLLM to set their own telemetry
@@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
             aembedding=aembedding,
             custom_endpoint=True,
             headers=headers,
+            drop_params=drop_params,
         )

litellm/llms/deepinfra/chat/transformation.py

@@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
         - Input:  {"role": "tool", "content": [{"type": "text", "text": "result"}]}
         - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
         """
-        for message in messages:
+        copied_messages: Final = [message.copy() for message in messages]
+        for message in copied_messages:
             if message.get("role") == "tool":
                 content = message.get("content")
 
@@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
                         # For complex content, serialize the entire array as JSON string
                         message["content"] = json.dumps(content)
 
-        return messages
+        return copied_messages
 
     @overload
     def _transform_messages(

litellm/llms/fireworks_ai/chat/transformation.py

@@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionToolParam,
+    ChatCompletionToolParamFunctionChunk,
     OpenAIChatCompletionToolParam,
 )
 from litellm.types.utils import (
@@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
     OpenAIGPTConfig,
 )
+from ...openai.common_utils import without_cache_control
 from ..common_utils import (
     FIREROUTER,
     FireworksAIException,
@@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
         return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
 
     def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
-        for tool in tools:
-            if tool.get("type") != "function":
-                continue
-            function = tool["function"]
-            function.pop("strict", None)
-            params = function.get("parameters")
-            if isinstance(params, dict):
-                unpack_legacy_defs(params)
-        return tools
+        return [self._fireworks_tool(tool) for tool in tools]
+
+    @staticmethod
+    def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
+        if tool.get("type") != "function":
+            return tool
+        function: Final = tool["function"]
+        params: Final = function.get("parameters")
+        unstrict_function: Final = cast(  # cast-ok: the same function definition minus `strict`
+            ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
+        )
+        new_function: Final[ChatCompletionToolParamFunctionChunk] = (
+            {**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
+            if isinstance(params, dict)
+            else unstrict_function
+        )
+        return {**tool, "function": new_function}
 
     def _transform_messages_helper(
         self, messages: list[AllMessageValues], model: str, litellm_params: dict
@@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
         """
         Strip fields not permitted by FireworksAI from messages.
         """
-        from litellm.litellm_core_utils.prompt_templates.common_utils import (
-            filter_value_from_dict,
-        )
-
         supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
         for message in messages:
             if message["role"] == "user":
@@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
                                 model=model,
                                 llm_provider="fireworks_ai",
                             )
-            filter_value_from_dict(cast(dict, message), "cache_control")
 
-        return [strip_litellm_response_only_fields(message) for message in messages]
+        return [
+            strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
+            for message in messages
+        ]
 
     # Cached index of fireworks_ai/* entries from litellm.model_cost. Building
     # this index requires a full scan of model_cost (tens of thousands of

litellm/llms/groq/chat/transformation.py

@@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
     def _transform_messages(
         self, messages: list[AllMessageValues], model: str, is_async: bool = False
     ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
-        for idx, message in enumerate(messages):
+        transformed_messages: Final = list(messages)
+        for idx, message in enumerate(transformed_messages):
             """
             1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
             """
@@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
                 for k, v in _message.items():
                     if v is not None:
                         new_message[k] = v
-                messages[idx] = strip_litellm_internal_message_fields(new_message)
+                transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
 
         if is_async:
-            return super()._transform_messages(messages=messages, model=model, is_async=True)
+            return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
         else:
-            return super()._transform_messages(messages=messages, model=model, is_async=False)
+            return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
 
     def transform_messages(
         self,

litellm/llms/hosted_vllm/chat/transformation.py

@@ -2,6 +2,7 @@
 Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
 """
 
+import copy
 import json
 from collections.abc import Coroutine
 from typing import Final, Literal, cast, overload
@@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
     ) -> dict:
         _tools = non_default_params.pop("tools", None)
         if _tools is not None:
-            _tools = remove_additional_properties(_tools)
+            _tools = remove_additional_properties(copy.deepcopy(_tools))  # pyright: ignore[reportUnknownArgumentType]  # tools arrive untyped through the bare-dict params
             _tools = remove_strict_from_schema(_tools)
             if isinstance(_tools, list):
                 _tools = self._convert_custom_tools_to_function_tools(_tools)

litellm/llms/mistral/chat/transformation.py

@@ -25,16 +25,16 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     strip_litellm_internal_message_fields,
     strip_none_values_from_message,
 )
+from litellm.litellm_core_utils.reasoning_effort_capability import (
+    declared_reasoning_efforts_for_model,
+    nearest_declared_reasoning_effort,
+)
 from litellm.litellm_core_utils.reasoning_effort_utils import OPAQUE_VALUE, warn_dropped_reasoning
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
     OpenAIGPTConfig,
 )
-from litellm.router_utils.reasoning_effort_capability import (
-    declared_reasoning_efforts_for_model,
-    nearest_declared_reasoning_effort,
-)
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.mistral import (
     MistralCacheKeyParams,
@@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
         if _name is not None:
             # Remove name if not a tool message
             if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
-                message.pop("name", None)
+                message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
+                return cast(AllMessageValues, message_without_name)  # cast-ok: the same message minus name
 
         return message
 

litellm/llms/oci/chat/transformation.py

@@ -21,6 +21,7 @@ from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
+from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.litellm_core_utils.url_utils import join_api_base
@@ -54,7 +55,6 @@ from litellm.llms.oci.common_utils import (
     sign_oci_request,
     validate_oci_environment,
 )
-from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.types.llms.oci import (
     CohereChatRequest,
     OCIChatRequestPayload,

litellm/llms/openai/chat/gpt_transformation.py

@@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
-from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     extract_reasoning_content,
     handle_invalid_parallel_tool_calls,
@@ -64,7 +64,7 @@ from litellm.types.utils import (
 )
 from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, without_cache_control
 from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
 
 if TYPE_CHECKING:
@@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
     return value if isinstance(value, str) else None
 
 
+def _drops_params(litellm_params: Mapping[str, object]) -> bool:
+    return drops_unsupported_params(litellm_params.get("drop_params"))
+
+
+def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
+    base_model: Final = litellm_params.get("base_model")
+    return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
+
+
+def _claude_messages_within_cache_control_cap(
+    model: str,
+    messages: Sequence[AllMessageValues],
+    tools: Sequence["ChatCompletionToolParam"] | None,
+    litellm_params: Mapping[str, object],
+    request_params: Mapping[str, object],
+) -> Sequence[AllMessageValues]:
+    if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
+        return messages
+    from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
+
+    return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
+
+
 def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
     try:
         extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         return content_copy
 
     async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
-        file_id: Final = content_item.get("file_id")
+        content_copy: Final = content_item.copy()
+        file_id: Final = content_copy.get("file_id")
         if file_id is not None:  # check for file id being url done in _handle_pdf_url
             base64_data: Final = await async_convert_url_to_base64(file_id)
-            content_item["file_data"] = base64_data
-            content_item["filename"] = "my_file.pdf"
-            content_item.pop("file_id")
-        return content_item
+            content_copy["file_data"] = base64_data
+            content_copy["filename"] = "my_file.pdf"
+            content_copy.pop("file_id")
+        return content_copy
 
     def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
         file_data: Final = content_item.get("file_data")
         filename: Final = content_item.get("filename")
         if file_data is not None and filename is None:
-            content_item["filename"] = "my_file.pdf"
+            return {**content_item, "filename": "my_file.pdf"}
         return content_item
 
+    @staticmethod
+    def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
+        image_url: Final = content_item["image_url"]
+        if isinstance(image_url, str):
+            return {**content_item, "image_url": {"url": image_url}}
+        if not isinstance(image_url, dict):
+            return content_item
+        new_image_url_obj: Final = cast(
+            ChatCompletionImageUrlObject,
+            {k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
+        )
+        return {**content_item, "image_url": new_image_url_obj}
+
+    @staticmethod
+    def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
+        litellm_specific_params: Final = {"format", "detail", "video_metadata"}
+        file_obj: Final = content_item.get("file")
+        if file_obj is None:
+            raise litellm.BadRequestError(
+                message="Content block has type='file' but is missing the required 'file' field",
+                model=None,
+                llm_provider="openai",
+            )
+        new_file_obj: Final = ChatCompletionFileObjectFile(
+            **{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
+        )
+        return {**content_item, "file": new_file_obj}
+
     def _apply_common_transform_content_item(
         self,
         content_item: OpenAIMessageContentListBlock,
     ) -> OpenAIMessageContentListBlock:
-        litellm_specific_params: Final = {"format", "detail", "video_metadata"}
         if content_item.get("type") == "image_url":
-            content_item = cast(ChatCompletionImageObject, content_item)
-            if isinstance(content_item["image_url"], str):
-                content_item["image_url"] = {
-                    "url": content_item["image_url"],
-                }
-            elif isinstance(content_item["image_url"], dict):
-                new_image_url_obj: Final = cast(
-                    ChatCompletionImageUrlObject,
-                    {k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
-                )
-                content_item["image_url"] = new_image_url_obj
-        elif content_item.get("type") == "file":
-            content_item = cast(ChatCompletionFileObject, content_item)
-            file_obj: Final = content_item.get("file")
-            if file_obj is None:
-                raise litellm.BadRequestError(
-                    message="Content block has type='file' but is missing the required 'file' field",
-                    model=None,
-                    llm_provider="openai",
-                )
-            new_file_obj: Final = ChatCompletionFileObjectFile(
-                **{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
-            )
-            content_item["file"] = new_file_obj
-
+            return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
+        if content_item.get("type") == "file":
+            return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
         return content_item
 
     def _transform_content_item(
@@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
 
         async def _async_transform():
-            for message in hoisted_messages:
-                message_content = message.get("content")
-                message_role = message.get("role")
-
-                if message_role == "user" and message_content and isinstance(message_content, list):
-                    message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
-                    for i, content_item in enumerate(message_content_types):
-                        message_content_types[i] = await self._async_transform_content_item(
-                            cast(OpenAIMessageContentListBlock, content_item),
-                        )
-            return hoisted_messages
+            return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
 
         if is_async:
             return _async_transform()
-        else:
-            for message in hoisted_messages:
-                message_content = message.get("content")
-                message_role = message.get("role")
-                if message_role == "user" and message_content and isinstance(message_content, list):
-                    message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
-                    for i, content_item in enumerate(message_content):
-                        message_content_types[i] = self._transform_content_item(
-                            cast(OpenAIMessageContentListBlock, content_item)
-                        )
-            return hoisted_messages
+        return [self._user_message_for_openai(message) for message in hoisted_messages]
+
+    @staticmethod
+    def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
+        message_content: Final = message.get("content")
+        if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
+            return None
+        return cast(Sequence[OpenAIMessageContentListBlock], message_content)
+
+    def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
+        parts: Final = self._user_content_parts(message)
+        if parts is None:
+            return message
+        return cast(  # cast-ok: the same user message with each content part rebuilt for OpenAI
+            AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
+        )
+
+    async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
+        parts: Final = self._user_content_parts(message)
+        if parts is None:
+            return message
+        return cast(  # cast-ok: the same user message with each content part rebuilt for OpenAI
+            AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
+        )
 
     def transform_messages(
         self,
@@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         messages: list[AllMessageValues],
         tools: list["ChatCompletionToolParam"] | None = None,
     ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
-        from litellm.litellm_core_utils.prompt_templates.common_utils import (
-            filter_value_from_dict,
-        )
         from litellm.types.llms.openai import ChatCompletionToolParam
 
-        for i, message in enumerate(messages):
-            messages[i] = cast(
-                AllMessageValues,
-                filter_value_from_dict(message, "cache_control"),
-            )
-        if tools is not None:
-            for i, tool in enumerate(tools):
-                tools[i] = cast(
-                    ChatCompletionToolParam,
-                    filter_value_from_dict(tool, "cache_control"),
-                )
-        return messages, tools
+        new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
+        new_tools: Final = (
+            [cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
+            if tools is not None
+            else None
+        )
+        return new_messages, new_tools
 
     def translate_developer_role_to_system_role(
         self,
@@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         tools: Sequence["ChatCompletionToolParam"] | None,
         prompt_cache_options: object,
         litellm_params: Mapping[str, object],
+        request_params: Mapping[str, object],
     ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
         """
         A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             model, custom_llm_provider, api_base, prompt_cache_options
         )
         if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
-            return messages, tools
-        drops_breakpoints: Final = not reads_breakpoints and (
-            litellm.drop_params is True or litellm_params.get("drop_params") is True
-        )
+            return (
+                _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
+                tools,
+            )
+        drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
         renamed: Final = (
             [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
             if reads_breakpoints
@@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             if drops_breakpoints
             else list(tools)
         )
-        return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
+        sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
+            model=model, messages=renamed, tools=kept_tools
+        )
+        return (
+            _claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
+            sent_tools,
+        )
 
     def _sanitized_tools_update_for_openai(
         self,
@@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             tools=optional_params.get("tools", []),
             prompt_cache_options=_prompt_cache_options(optional_params),
             litellm_params=litellm_params,
+            request_params=optional_params,
         )
         if tools is not None and len(tools) > 0:
             optional_params["tools"] = tools
@@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             tools=optional_params.get("tools", []),
             prompt_cache_options=_prompt_cache_options(optional_params),
             litellm_params=litellm_params,
+            request_params=optional_params,
         )
         if tools is not None and len(tools) > 0:
             optional_params["tools"] = tools

litellm/llms/openai/chat/o_series_transformation.py

@@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
         - role: system ==> translate to role 'user'
         """
         _supports_system_messages: Final = supports_system_messages(model, "openai")
-        for i, message in enumerate(messages):
+        transformed_messages: Final = list(messages)
+        for i, message in enumerate(transformed_messages):
             if message["role"] == "system" and not _supports_system_messages:
                 new_message = ChatCompletionUserMessage(content=message["content"], role="user")
-                messages[i] = new_message  # Replace the old message with the new one
+                transformed_messages[i] = new_message  # Replace the old message with the new one
 
         if is_async:
-            return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
+            return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
         else:
-            return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
+            return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

litellm/llms/openai/common_utils.py

@@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
 """
 
 import asyncio
+import copy
 import hashlib
 import inspect
 import json
@@ -31,7 +32,7 @@ if TYPE_CHECKING:
 import litellm
 from litellm.exceptions import error_response_without_credentials
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
 from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 
 
+def _is_plain_dict(value: object) -> bool:
+    return isinstance(value, dict)
+
+
+def _is_list(value: object) -> bool:
+    return isinstance(value, list)
+
+
+def _contains_cache_control(value: object) -> bool:
+    pending: Final[list[object]] = [value]  # mutable-ok: a local worklist, never returned or stored
+    while pending:
+        node = pending.pop()
+        mapping = as_str_mapping(node)
+        if mapping is not None:
+            if "cache_control" in mapping:
+                return True
+            pending.extend(mapping.values())
+        elif _is_list(node):
+            pending.extend(iter_items(node))
+    return False
+
+
+def without_cache_control(item: object) -> object:
+    """Return `item` with every `cache_control` key removed, without touching the original.
+
+    `filter_value_from_dict` deletes the key in place and recurses into nested dicts and
+    lists, so the caller's own message, tool or input object has to be copied first: it
+    may reuse the same list on a provider that does support prompt caching. The copy is
+    skipped when there is nothing to strip, which keeps the object identity the
+    pass-through paths rely on and avoids a deep copy on every request.
+    """
+    from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
+
+    if not _is_plain_dict(item) or not _contains_cache_control(item):
+        return item
+    copied: Final = copy.deepcopy(item)
+    if isinstance(copied, dict):
+        filter_value_from_dict(copied, "cache_control")
+    return copied
+
+
 def _get_client_init_params(cls: type) -> tuple[str, ...]:
     """Extract __init__ parameter names (excluding 'self') from a class."""
     return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

litellm/llms/openai/openai.py

@@ -16,6 +16,7 @@ from openai._base_client import make_request_options
 from openai._constants import RAW_RESPONSE_HEADER
 from openai._legacy_response import LegacyAPIResponse
 from openai._types import RequestOptions
+from openai.pagination import SyncCursorPage
 from openai.types import CreateEmbeddingResponse
 from openai.types.beta.assistant_deleted import AssistantDeleted
 from openai.types.chat import ChatCompletion, ChatCompletionChunk
@@ -40,6 +41,7 @@ from litellm.types.utils import (
     LiteLLMBatch,
     ModelResponse,
     ModelResponseStream,
+    openai_batch_status,
 )
 from litellm.utils import (
     CustomStreamWrapper,
@@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
         return response
 
 
+def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
+    return [
+        batch.model_copy(
+            update={
+                "object": batch.object or "batch",
+                "status": openai_batch_status(batch.status),
+            }
+        )
+        for batch in batches
+    ]
+
+
 class OpenAIBatchesAPI(BaseLLM):
     """
     OpenAI methods to support for batches
@@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
     ):
         verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
         response: Final = await openai_client.batches.list(after=after, limit=limit)
-        return response
+        return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
 
     def list_batches(
         self,
@@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
         organization: str | None,
         after: str | None = None,
         limit: int | None = None,
-        client: OpenAI | None = None,
+        client: OpenAI | AsyncOpenAI | None = None,
         litellm_params: Mapping[str, object] | None = None,
     ):
         openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
                 )
             return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
         response: Final = openai_client.batches.list(after=after, limit=limit)
-        return response
+        if not isinstance(response, SyncCursorPage):
+            return response
+        return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
 
 
 class OpenAIAssistantsAPI(BaseLLM):

litellm/llms/openai/responses/transformation.py

@@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
 
-from ..common_utils import OpenAIError
+from ..common_utils import OpenAIError, without_cache_control
 from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
 
 OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
             input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
             tools=mapped_tools,
         )
-        if marker_safe_input is not validated_input:
-            self._strip_caller_input_as_if_unmarked(
-                model=model,
-                caller_input=validated_input,
-                reads_breakpoints=reads_breakpoints,
-                maps_anthropic_tool_search=maps_anthropic_tool_search,
-            )
         object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
         sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
             model=model, tools=object_schema_tools, litellm_params=litellm_params
@@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
             return marked_input
         return input_without_unanswered_anthropic_tool_search_calls(marked_input)
 
-    def _strip_caller_input_as_if_unmarked(
-        self,
-        model: str,
-        caller_input: str | ResponseInputParam,
-        reads_breakpoints: bool,
-        maps_anthropic_tool_search: bool,
-    ) -> None:
-        self.remove_cache_control_flag_from_input_and_tools(
-            model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
-        )
-
     def _input_with_wire_safe_prompt_cache_breakpoints(
         self,
         model: str,
@@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         the chat path. Strips Anthropic-only `cache_control` markers from
         Responses API input content blocks and tools.
 
-        `filter_value_from_dict` mutates each dict in place, so the same
-        objects are returned.
+        `filter_value_from_dict` deletes the key in place and recurses, so each
+        item is copied first: the caller keeps its own input list and may reuse
+        it on a provider that does support prompt caching.
         """
-        from litellm.litellm_core_utils.prompt_templates.common_utils import (
-            filter_value_from_dict,
+        new_input: Final = (
+            cast(  # cast-ok: the comprehension rebuilds the input item for item
+                ResponseInputParam,
+                [without_cache_control(item) for item in input],
+            )
+            if isinstance(input, list)
+            else input
         )
 
-        if isinstance(input, list):
-            for item in input:
-                if isinstance(item, dict):
-                    filter_value_from_dict(cast(dict, item), "cache_control")
+        new_tools: Final = (
+            cast(  # cast-ok: the comprehension rebuilds the tool list item for item
+                Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
+                [without_cache_control(tool) for tool in tools],
+            )
+            if tools is not None
+            else None
+        )
 
-        if tools is not None:
-            for tool in tools:
-                if isinstance(tool, dict):
-                    filter_value_from_dict(cast(dict, tool), "cache_control")
-
-        return input, tools
+        return new_input, new_tools
 
     def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
         if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

litellm/llms/openai_like/README.md

@@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
 
 If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
 
+When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
+
+```json
+{
+  "your_provider": {
+    "supported_endpoints": ["/v1/chat/completions"],
+    "model_supported_endpoints": {
+      "model-a": ["/v1/messages"],
+      "model-b": ["/v1/responses"]
+    }
+  }
+}
+```
+
 ### How It Works
 
 1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`

litellm/llms/openai_like/embedding/handler.py

@@ -3,11 +3,16 @@
 ## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
 
 import json
+import re
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
 
 from ..common_utils import OpenAILikeBase, OpenAILikeError
 
+_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
+_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_TRANSPORT_HEADER_NAMES: Final = frozenset(
+    {
+        "host",
+        "content-length",
+        "transfer-encoding",
+        "connection",
+        "keep-alive",
+        "proxy-connection",
+        "te",
+        "trailer",
+        "upgrade",
+        "expect",
+    }
+)
+_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
+_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
+
+
+def _header_text(name: str, value: object) -> str | None:
+    if isinstance(value, bool) or not isinstance(value, str | int | float):
+        return None
+    if _HEADER_NAME.fullmatch(name) is None:
+        return None
+    return str(value).strip(" \t\r\n")
+
+
+def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
+    caller_set: Final = {
+        name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
+    }
+    texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
+    refused: Final = sorted(name for name, text in texts.items() if text is None)
+    if refused and not drop_params:
+        raise litellm.UnsupportedParamsError(
+            message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
+            "letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
+            "To drop them from the call, set `litellm.drop_params = True`.",
+            model=model,
+        )
+    named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
+    unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
+    if unsendable:
+        verbose_logger.warning(
+            "extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
+            unsendable,
+            model,
+        )
+    return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
+
 
 class OpenAILikeEmbeddingHandler(OpenAILikeBase):
     def __init__(self, **kwargs):
@@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
         aembedding=None,
         custom_endpoint: bool | None = None,
         headers: dict | None = None,
+        drop_params: bool = False,
     ) -> EmbeddingResponse:
         api_base, headers = self._validate_environment(
             api_base=api_base,
@@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
             custom_endpoint=custom_endpoint,
         )
         model = model
-        filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
-        data: Final = {"model": model, "input": input, **filtered_optional_params}
+        data: Final = {
+            "model": model,
+            "input": input,
+            **{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
+        }
+        extra_headers: Final = _extra_headers_as_text(
+            _EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
+        )
+        key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
+        extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
+        request_headers: Final = {
+            **{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
+            **extra_headers,
+        }
 
         ## LOGGING
         logging_obj.pre_call(
@@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
                 api_key=api_key,
                 timeout=timeout,
                 client=client,
-                headers=headers,
+                headers=request_headers,
             )
         if client is None or isinstance(client, AsyncHTTPHandler):
             self.client = HTTPHandler(timeout=timeout)
@@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
         try:
             response: Final = self.client.post(
                 api_base,
-                headers=headers,
+                headers=request_headers,
                 data=json.dumps(data),
             )
 

litellm/llms/openai_like/json_loader.py

@@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from pathlib import Path
 from typing import Final
 
@@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
 
 _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
 _SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
+_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
 
 
 class SimpleProviderConfig:
@@ -30,7 +31,10 @@ class SimpleProviderConfig:
         self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
             data.get("special_handling", {})
         )
-        self.supported_endpoints = data.get("supported_endpoints", [])
+        self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
+        self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
+            data.get("model_supported_endpoints", {})
+        )
         self.provider_info: ProviderSpecificModelInfo = (
             _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
         )
@@ -90,6 +94,23 @@ class JSONProviderRegistry:
             return False
         return "/v1/responses" in provider.supported_endpoints
 
+    @classmethod
+    def supports_embeddings(cls, slug: str) -> bool:
+        provider: Final = cls._providers.get(slug)
+        if provider is None:
+            return False
+        return "/v1/embeddings" in provider.supported_endpoints
+
+    @classmethod
+    def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
+        provider: Final = cls._providers.get(slug)
+        if provider is None:
+            return False
+        model_endpoints: Final = (
+            provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
+        )
+        return endpoint in provider.supported_endpoints or endpoint in model_endpoints
+
     @classmethod
     def list_providers(cls) -> list:
         """List all registered provider slugs"""

litellm/llms/openai_like/providers.json

@@ -211,7 +211,7 @@
     "param_mappings": {
       "max_completion_tokens": "max_tokens"
     },
-    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
   },
   "scx-ai": {
     "base_url": "https://api.scx.ai/v1",
@@ -231,6 +231,16 @@
     "api_base_env": "PRISM_API_BASE",
     "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
   },
+  "hostinger_router": {
+    "base_url": "https://router.hostinger.com/v1",
+    "api_key_env": "HOSTINGER_ROUTER_API_KEY",
+    "api_base_env": "HOSTINGER_ROUTER_API_BASE",
+    "supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
+    "model_supported_endpoints": {
+      "claude-sonnet-5": ["/v1/messages"],
+      "gpt-5.2": ["/v1/responses"]
+    }
+  },
   "sail": {
     "base_url": "https://api.sailresearch.com/v1",
     "api_key_env": "SAIL_API_KEY",

litellm/llms/together_ai/chat/transformation.py

@@ -18,7 +18,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.prompt_templates.common_utils import strip_litellm_response_only_fields
-from litellm.router_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
+from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.types.llms.openai import AllMessageValues
 from litellm.utils import supports_function_calling, supports_reasoning, supports_response_schema
 

litellm/llms/vertex_ai/context_caching/transformation.py

@@ -110,38 +110,23 @@ def separate_cached_messages(
     messages: list[AllMessageValues],
 ) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
     """
-    Returns separated cached and non-cached messages.
-
-    Args:
-        messages: List of messages to be separated.
-
-    Returns:
-        Tuple containing:
-        - cached_messages: List of cached messages.
-        - non_cached_messages: List of non-cached messages.
+    Returns the first continuous block of cached messages, plus every system message once
+    any message is cached, since generateContent rejects a system_instruction beside a
+    cachedContent, and the remaining messages.
     """
-    cached_messages: list[AllMessageValues] = []
-    non_cached_messages: list[AllMessageValues] = []
+    filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
+    if not filtered_messages:
+        return [], messages
 
-    # Extract cached messages and their indices
-    filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
-    for idx, message in enumerate(messages):
-        if is_cached_message(message=message):
-            filtered_messages.append((idx, message))
-
-    # Validate only one block of continuous cached messages
     last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
-    # Separate messages based on the block of cached messages
-    if filtered_messages and last_continuous_block_idx is not None:
-        first_cached_idx: Final = filtered_messages[0][0]
-        last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
-
-        cached_messages = messages[first_cached_idx : last_cached_idx + 1]
-        non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
-    else:
-        non_cached_messages = messages
-
-    return cached_messages, non_cached_messages
+    cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
+    goes_in_cache: Final = tuple(
+        idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
+    )
+    return (
+        [message for message, cached in zip(messages, goes_in_cache) if cached],
+        [message for message, cached in zip(messages, goes_in_cache) if not cached],
+    )
 
 
 def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

litellm/llms/watsonx/chat/transformation.py

@@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
 Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
 """
 
+import copy
 from typing import Final
 
 from litellm import verbose_logger
@@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
         _tools = non_default_params.pop("tools", None)
         if _tools is not None:
             # remove 'additionalProperties' from tools
-            _tools = remove_additional_properties(_tools)
+            _tools = remove_additional_properties(copy.deepcopy(_tools))  # pyright: ignore[reportUnknownArgumentType]  # tools arrive untyped through the bare-dict params
             # remove 'strict' from tools
             _tools = remove_strict_from_schema(_tools)
         if _tools is not None:

litellm/llms/xai/chat/transformation.py

@@ -1,3 +1,4 @@
+import copy
 from collections.abc import AsyncIterator, Iterator, Mapping
 from types import MappingProxyType
 from typing import Any, Final
@@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
                 tools = [
                     filtered_tool
                     for filtered_tool in (
-                        _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
+                        _FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict"))  # pyright: ignore[reportUnknownArgumentType]  # tools arrive untyped through the bare-dict params
+                        for tool in value
                     )
                     if filtered_tool is not None
                 ]

litellm/main.py

@@ -113,6 +113,11 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     messages_without_pdf_file_parts,
 )
 from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
+from litellm.litellm_core_utils.reasoning_effort_capability import (
+    model_group_from_metadata,
+    reasoning_effort_needs_allowing,
+    resolve_chat_reasoning,
+)
 from litellm.litellm_core_utils.request_timeout_resolver import (
     get_configured_request_timeout,
 )
@@ -144,11 +149,6 @@ from litellm.llms.vertex_ai.common_utils import (
 )
 from litellm.realtime_api.main import realtime_health_check
 from litellm.responses.mcp.tool_precheck import may_name_litellm_mcp_gateway
-from litellm.router_utils.reasoning_effort_capability import (
-    model_group_from_metadata,
-    reasoning_effort_needs_allowing,
-    resolve_chat_reasoning,
-)
 from litellm.secret_managers.main import get_secret_bool, get_secret_str
 from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
 from litellm.types.completion import (
@@ -6809,6 +6809,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
         )
 
 
+def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if not configured:
+        return {}
+    if isinstance(configured, Mapping):
+        return dict(cast(Mapping[str, object], configured))  # cast-ok: header names from untyped request JSON
+    try:
+        return dict(cast(Iterable[tuple[str, object]], configured))  # cast-ok: name and value pairs from request JSON
+    except (TypeError, ValueError):
+        if drop_params:
+            return {}
+        raise litellm.UnsupportedParamsError(
+            message=f"{param} must be a JSON object of header names to values. "
+            "To drop it from the call, set `litellm.drop_params = True`.",
+            model=model,
+        )
+
+
 # fmt: off
 
 # Overload for when aembedding=True (returns coroutine)
@@ -6923,11 +6940,10 @@ def embedding(
     azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
     aembedding: Final[bool | None] = kwargs.get("aembedding", None)
     extra_headers: Final = kwargs.get("extra_headers", None)
-    headers = kwargs.get("headers", None) or extra_headers
-    if headers is None:
-        headers = {}
-    if extra_headers is not None:
-        headers.update(extra_headers)
+    headers = {
+        **_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
+        **_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
+    }
     # Inject proxy auth headers if configured
     if litellm.proxy_auth is not None:
         try:
@@ -7091,7 +7107,7 @@ def embedding(
             # set API KEY
             api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
 
-            if headers is not None and headers != {}:
+            if headers:
                 optional_params["extra_headers"] = headers
 
             requested_encoding_format: Final = (
@@ -7183,14 +7199,15 @@ def embedding(
             custom_llm_provider == "openai_like"
             or custom_llm_provider == "llamafile"
             or custom_llm_provider == "lm_studio"
+            or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
         ):
             api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
 
             # set API KEY
-            if api_key is None:
-                api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
+            if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
+                api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
 
-            if headers is not None and headers != {}:
+            if headers:
                 optional_params["extra_headers"] = headers
 
             ## EMBEDDING CALL
@@ -7205,10 +7222,9 @@ def embedding(
                 optional_params=optional_params,
                 client=client,
                 aembedding=aembedding,
+                drop_params=drop_unsupported_params,
             )
         elif custom_llm_provider == "oci":
-            if headers is None:
-                headers = {}
             response = base_llm_http_handler.embedding(
                 model=model,
                 input=input,
@@ -7233,11 +7249,6 @@ def embedding(
                 or litellm.api_key
             )
 
-            # Use the merged headers variable (already merged at the top of the function)
-            # Don't overwrite it with just extra_headers
-            if headers is None:
-                headers = {}
-
             response = base_llm_http_handler.embedding(
                 model=model,
                 input=input,

litellm/model_prices_and_context_window_backup.json

@@ -25302,6 +25302,38 @@
         "cache_creation_input_token_cost": 1e-06,
         "prompt_cache_min_tokens": 2048
     },
+    "eu.anthropic.claude-haiku-4-5-20251001-v1:0": {
+        "cache_creation_input_token_cost": 1.375e-06,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-06,
+        "cache_read_input_token_cost": 1.1e-07,
+        "input_cost_per_token": 1.1e-06,
+        "deprecation_date": "2026-10-15",
+        "litellm_provider": "bedrock_converse",
+        "supports_tool_search": true,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 5.5e-06,
+        "source": "https://aws.amazon.com/bedrock/pricing/",
+        "supports_assistant_prefill": true,
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_native_structured_output": true,
+        "supports_parallel_tool_use_config": true,
+        "prompt_cache_min_tokens": 4096,
+        "input_cost_per_token_batches": 5.5e-07,
+        "output_cost_per_token_batches": 2.75e-06,
+        "token_count_model": {
+            "model": "anthropic.claude-haiku-4-5-20251001-v1:0",
+            "litellm_provider": "bedrock"
+        }
+    },
     "eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
         "input_cost_per_token": 3e-06,
         "litellm_provider": "bedrock",
@@ -39655,6 +39687,57 @@
             "/v1/images/generations"
         ]
     },
+    "hostinger_router/gpt-5.2": {
+        "cache_read_input_token_cost": 0.00000021,
+        "input_cost_per_token": 0.0000021,
+        "litellm_provider": "hostinger_router",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 0.0000168,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
+        "source": "https://router.hostinger.com/v1/models"
+    },
+    "hostinger_router/claude-sonnet-5": {
+        "cache_creation_input_token_cost": 3.3e-06,
+        "cache_creation_input_token_cost_above_1hr": 5.28e-06,
+        "cache_read_input_token_cost": 2.64e-07,
+        "input_cost_per_token": 2.64e-06,
+        "litellm_provider": "hostinger_router",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-05,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
+        "source": "https://router.hostinger.com/v1/models"
+    },
+    "hostinger_router/text-embedding-3-large": {
+        "input_cost_per_token": 1.716e-07,
+        "litellm_provider": "hostinger_router",
+        "max_input_tokens": 8191,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "source": "https://router.hostinger.com/v1/models"
+    },
+    "hostinger_router/text-embedding-3-small": {
+        "input_cost_per_token": 3e-08,
+        "litellm_provider": "hostinger_router",
+        "max_input_tokens": 8191,
+        "mode": "embedding",
+        "output_cost_per_token": 0.0,
+        "source": "https://router.hostinger.com/v1/models"
+    },
     "hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
         "input_cost_per_token": 1.2e-07,
         "litellm_provider": "hyperbolic",
@@ -51355,6 +51438,11 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
         "output_cost_per_token": 5e-06,
         "output_cost_per_token_batches": 2.5e-06,
         "regional_endpoint_uplift_multiplier": 1.1,
@@ -51383,6 +51471,11 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
         "output_cost_per_token": 5e-06,
         "output_cost_per_token_batches": 2.5e-06,
         "regional_endpoint_uplift_multiplier": 1.1,
@@ -52129,6 +52222,11 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
         "regional_endpoint_uplift_multiplier": 1.1,
@@ -52234,6 +52332,11 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 64000,
         "mode": "chat",
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
         "regional_endpoint_uplift_multiplier": 1.1,

litellm/provider_endpoints_support_backup.json

@@ -1087,6 +1087,32 @@
         "generateContent": false
       }
     },
+    "hostinger_router": {
+      "display_name": "Hostinger Router (`hostinger_router`)",
+      "url": "https://docs.litellm.ai/docs/providers/hostinger_router",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": true,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
+    },
     "crusoe": {
       "display_name": "Crusoe (`crusoe`)",
       "url": "https://docs.litellm.ai/docs/providers/crusoe",

litellm/proxy/_lazy_openapi_snapshot.json

@@ -58015,7 +58015,7 @@
     "paths": {
       "/toolset/{toolset_name}/mcp": {
         "delete": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
           "parameters": [
             {
@@ -58054,7 +58054,7 @@
           ]
         },
         "get": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
           "parameters": [
             {
@@ -58093,7 +58093,7 @@
           ]
         },
         "head": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
           "parameters": [
             {
@@ -58132,7 +58132,7 @@
           ]
         },
         "options": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
           "parameters": [
             {
@@ -58171,7 +58171,7 @@
           ]
         },
         "patch": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
           "parameters": [
             {
@@ -58210,7 +58210,7 @@
           ]
         },
         "post": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
           "parameters": [
             {
@@ -58249,7 +58249,7 @@
           ]
         },
         "put": {
-          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
+          "description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
           "operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
           "parameters": [
             {

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -431,6 +431,7 @@ async def _handle_stream_message(
     proxy_server_request: dict[str, object] | None = None,
     *,
     agent_extra_headers: dict[str, str] | None = None,
+    agent_name: str | None = None,
     user_api_key_dict: UserAPIKeyAuth | None = None,
     request_data: dict[str, object] | None = None,
     proxy_logging_obj: ProxyLogging | None = None,
@@ -504,6 +505,7 @@ async def _handle_stream_message(
                 metadata=metadata,
                 proxy_server_request=proxy_server_request,
                 agent_extra_headers=agent_extra_headers,
+                agent_name=agent_name,
             )
 
             if (
@@ -932,6 +934,7 @@ async def invoke_agent_a2a(
                 metadata=data.get("metadata", {}),
                 proxy_server_request=data.get("proxy_server_request"),
                 agent_extra_headers=agent_extra_headers,
+                agent_name=str(agent_name),
                 user_api_key_dict=user_api_key_auth_for_call,
                 request_data=data,
                 proxy_logging_obj=proxy_logging_obj,

litellm/proxy/auth/auth_checks.py

@@ -167,6 +167,7 @@ from litellm.repositories.table_repositories import (
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.router import Router
+from litellm.router_api import RegistryReader
 from litellm.types.proxy.auth.auth_checks import UserNotFoundError
 from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
 from litellm.types.router import DeploymentTypedDict
@@ -525,7 +526,7 @@ def _log_budget_lookup_failure(entity: str, error: Exception) -> None:
     )
 
 
-def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
+def _get_router_zero_cost_cache(llm_router: RegistryReader) -> dict[str, bool] | None:
     """
     Return the router's per-instance zero-cost cache, or ``None`` for objects
     that don't expose one (e.g. ``MagicMock`` stand-ins in unit tests).
@@ -541,7 +542,7 @@ def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
     return cache if isinstance(cache, dict) else None
 
 
-def is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) -> bool:
+def is_model_cost_zero(model: str | list[str] | None, llm_router: RegistryReader | None) -> bool:
     """
     Check if a model has zero cost (no configured pricing).
 
@@ -667,7 +668,7 @@ _NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({})
 _TEAM_GRANT_RELATIONS: Final[Mapping[str, object]] = MappingProxyType({"litellm_model_table": True})
 
 
-def _deployments_served_for(model: str, llm_router: "Router") -> Sequence[DeploymentTypedDict]:
+def _deployments_served_for(model: str, llm_router: RegistryReader) -> Sequence[DeploymentTypedDict]:
     """
     The deployments a request to ``model`` routes to, for the gates that waive budget checks.
 
@@ -4842,7 +4843,7 @@ async def get_skill_names_from_access_groups(
 
 def _resolve_all_team_model_sentinel_for_auth_check(
     models: list[str],
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     team_id: str | None,
 ) -> list[str]:
     if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
@@ -4856,7 +4857,7 @@ def _resolve_all_team_model_sentinel_for_auth_check(
 
 def check_model_access_helper(
     model: str,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     models: list[str],
     team_model_aliases: dict[str, str] | None = None,
     team_id: str | None = None,
@@ -4907,7 +4908,7 @@ _check_model_access_helper: Final = check_model_access_helper
 
 def _owning_team_names_for_team_deployment(
     model: str,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     team_id: str | None,
     team_model_aliases: Mapping[str, str] | None,
 ) -> tuple[str, ...]:
@@ -4925,7 +4926,7 @@ def _owning_team_names_for_team_deployment(
 
 def _can_object_call_model(
     model: str | list[str],
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     models: list[str],
     team_model_aliases: dict[str, str] | None = None,
     team_id: str | None = None,
@@ -5052,7 +5053,7 @@ def _resolve_team_alias(
 
 
 def _live_team_alias_target(
-    model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: Router | None
+    model: str, team_model_aliases: Mapping[str, str], team_id: str | None, llm_router: RegistryReader | None
 ) -> str:
     target: Final = team_model_aliases.get(model)
     if target is None:
@@ -5539,7 +5540,7 @@ async def can_key_call_model(
     model: str | list[str],
     llm_model_list: Sequence[object] | None,
     valid_token: UserAPIKeyAuth,
-    llm_router: litellm.Router | None,
+    llm_router: RegistryReader | None,
     prisma_client: DatabaseClient | None = None,
 ) -> Literal[True]:
     """
@@ -5618,7 +5619,7 @@ async def can_key_call_resolved_model(
     model: str,
     llm_model_list: Sequence[object] | None,
     valid_token: UserAPIKeyAuth,
-    llm_router: litellm.Router | None,
+    llm_router: RegistryReader | None,
 ) -> None:
     from litellm.proxy.proxy_server import (
         prisma_client,
@@ -5766,7 +5767,7 @@ def can_org_access_model(
 async def can_team_access_model(
     model: str | list[str],
     team_object: LiteLLM_TeamTable | None,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     team_model_aliases: dict[str, str] | None = None,
     key_model_aliases: Mapping[str, str] | None = None,
     prisma_client: DatabaseClient | None = None,
@@ -5869,7 +5870,7 @@ async def key_access_group_grants_model(
     model: str | list[str],
     valid_token: UserAPIKeyAuth | None,
     team_object: LiteLLM_TeamTable | None,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     key_model_aliases: Mapping[str, str] | None = None,
 ) -> bool:
     """
@@ -5905,7 +5906,7 @@ _key_access_group_grants_model: Final = key_access_group_grants_model
 def can_project_access_model(
     model: str | list[str],
     project_object: LiteLLM_ProjectTable,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     key_model_aliases: Mapping[str, str] | None = None,
 ) -> Literal[True]:
     """
@@ -5926,7 +5927,7 @@ def can_project_access_model(
 def can_customer_access_model(
     model: str | list[str],
     end_user_object: LiteLLM_EndUserTable,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     valid_token: UserAPIKeyAuth | None,
 ) -> Literal[True]:
     team_model_aliases: Final = team_model_aliases_for_auth_check(valid_token) if valid_token is not None else None
@@ -6089,7 +6090,7 @@ async def _team_grants_personal_key_model(
     model: str,
     team_object: LiteLLM_TeamTableCachedObj,
     valid_token: UserAPIKeyAuth,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     prisma_client: PrismaClient | None,
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
@@ -6127,7 +6128,7 @@ async def _loaded_teams_allow_model(
     model: str,
     teams: tuple[LiteLLM_TeamTableCachedObj | None, ...],
     valid_token: UserAPIKeyAuth,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     prisma_client: PrismaClient | None,
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
@@ -6393,7 +6394,7 @@ async def can_personal_key_call_model_via_teams(
     model: str | Sequence[str],
     user_object: LiteLLM_UserTable | None,
     valid_token: UserAPIKeyAuth,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     prisma_client: PrismaClient | None,
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
@@ -7185,7 +7186,7 @@ async def check_team_member_model_access(
     model: str | list[str],
     team_object: LiteLLM_TeamTable,
     valid_token: UserAPIKeyAuth,
-    llm_router: Router | None,
+    llm_router: RegistryReader | None,
     prisma_client: Optional["PrismaClient"],
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,

litellm/proxy/auth/fallback_budget.py

@@ -44,7 +44,7 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     _is_model_cost_zero,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     is_model_cost_zero,  # pyright: ignore[reportPrivateUsage]  # the zero-cost predicate the auth-time budget checks use; no public equivalent
 )
-from litellm.router import Router
+from litellm.router_api import RegistryReader, RoutingContext
 from litellm.types.llms.base import LiteLLMBaseModel
 
 
@@ -118,7 +118,7 @@ async def is_token_within_budget_for_model(
     *,
     model: str,
     valid_token: UserAPIKeyAuth,
-    llm_router: Router,
+    llm_router: RegistryReader,
     team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None,
 ) -> bool:
     """
@@ -187,19 +187,19 @@ class RouterFallbackBudgetCheck:
     is_enforced: Callable[[], bool]
     team_member_model_budget_limiter: _TeamMemberModelBudgetLimiter | None = None
 
-    async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
+    async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
         if not self.is_enforced():
             return True
         valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
         if valid_token is None:
             return True
-        budget_models: Final = tuple(dict.fromkeys((model, llm_router.registry().resolve(model))))
+        budget_models: Final = tuple(dict.fromkeys((model, ctx.registry().resolve(model))))
         for budget_model in budget_models:
             try:
                 if not await is_token_within_budget_for_model(
                     model=budget_model,
                     valid_token=valid_token,
-                    llm_router=llm_router,
+                    llm_router=ctx,
                     team_member_model_budget_limiter=self.team_member_model_budget_limiter,
                 ):
                     return False

litellm/proxy/auth/fallback_model_access.py

@@ -18,7 +18,7 @@ from pydantic import ValidationError
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool
-from litellm.router import Router
+from litellm.router_api import RegistryReader, RoutingContext
 from litellm.search import asearch
 from litellm.types.llms.base import LiteLLMBaseModel
 
@@ -31,7 +31,7 @@ class _FallbackAccessSettings(LiteLLMBaseModel):
     enforce_fallback_model_access: bool = True
 
 
-async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: Router) -> bool:
+async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAuth, llm_router: RegistryReader) -> bool:
     try:
         await can_key_call_resolved_model(
             model=model,
@@ -98,7 +98,7 @@ class RouterFallbackAccessCheck:
 
     is_enforced: Callable[[], bool]
 
-    async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
+    async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], ctx: RoutingContext) -> bool:
         valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
         if valid_token is None:
             return True
@@ -106,7 +106,7 @@ class RouterFallbackAccessCheck:
             return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token)
         if not self.is_enforced():
             return True
-        return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
+        return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=ctx)
 
 
 router_fallback_access_check: Final = RouterFallbackAccessCheck(is_enforced=_enforced_by_general_settings)

litellm/proxy/auth/model_checks.py

@@ -14,13 +14,13 @@ from typing import Final, TypeVar
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter
 from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.proxy._types import SpecialModelNames, UserAPIKeyAuth
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
 from litellm.responses.main import strip_responses_routing_prefix
 from litellm.router import Router
-from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
-from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+from litellm.router_api import fallback_chain
 from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager, get_valid_models, potential_model_names_in_lookup_order
@@ -708,7 +708,7 @@ def get_all_fallbacks(
 
     try:
         # Use existing function to get fallback model group
-        fallback_model_group, _ = get_fallback_model_group(fallbacks=fallbacks_config, model_group=model)
+        fallback_model_group, _ = fallback_chain(fallbacks=fallbacks_config, model_group=model)
 
         if fallback_model_group is None:
             return []

litellm/proxy/auth/user_api_key_auth.py

@@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
 from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
 from litellm.models.end_user import LiteLLM_EndUserTable
 from litellm.proxy._types import *
 from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
@@ -182,13 +183,7 @@ from litellm.proxy.utils import (
 from litellm.repositories.table_repositories import JWTKeyMappingRepository, TeamMembershipRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.repositories.verification_token_repository import VerificationTokenRepository
-from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
-from litellm.router_utils.fallback_event_handlers import (
-    PROVIDER_SCOPED_RESOURCE_KEYS,
-    fallback_lookup_groups,
-    fallbacks_disabled_for_request,
-    get_fallback_model_group_for_lookup_groups,
-)
+from litellm.router_api import fallbacks_disabled, references_provider_scoped_resource, request_fallback_chain
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.services import ServiceTypes
 
@@ -811,11 +806,7 @@ async def _check_team_member_model_budget_for_model(
             model=model_name,
         )
     except litellm.BudgetExceededError:
-        if (
-            request_data.get("model") != model_name
-            or llm_router is None
-            or fallbacks_disabled_for_request(kwargs=request_data)
-        ):
+        if request_data.get("model") != model_name or llm_router is None or fallbacks_disabled(request_data):
             raise
         fallback_succeeded: Final = await _try_team_member_model_budget_fallback(
             user_id=user_id,
@@ -857,9 +848,9 @@ async def _try_team_member_model_budget_fallback(
     user_api_key_cache: UserApiKeyCache | None = None,
     proxy_logging_obj: ProxyLogging | None = None,
 ) -> bool:
-    if request.headers.get("content-type", "").lower().startswith("multipart/form-data") or any(
-        request_data.get(key) for key in PROVIDER_SCOPED_RESOURCE_KEYS
-    ):
+    if request.headers.get("content-type", "").lower().startswith(
+        "multipart/form-data"
+    ) or references_provider_scoped_resource(request_data):
         return False
 
     from litellm.proxy import proxy_server
@@ -896,10 +887,7 @@ async def _try_team_member_model_budget_fallback(
     if router_fallbacks is None:
         return False
 
-    fallback_models, _ = get_fallback_model_group_for_lookup_groups(
-        fallbacks=list(router_fallbacks),
-        lookup_groups=fallback_lookup_groups(kwargs=request_data, model_group=model_name),
-    )
+    fallback_models, _ = request_fallback_chain(list(router_fallbacks), request_data, model_name)
     fallback_targets: Final[Sequence[object]] = fallback_models or ()
     fallback_candidates: Final[tuple[str, ...]] = tuple(
         model for target in fallback_targets if (model := _budget_fallback_target_model(target)) is not None

litellm/proxy/bug_report_config.py

@@ -19,8 +19,8 @@ from litellm.litellm_core_utils.bug_report import (
     build_bug_report,
     build_environment_report,
 )
+from litellm.litellm_core_utils.routing_groups import VALID_ROUTING_STRATEGIES
 from litellm.proxy._types import ConfigGeneralSettings
-from litellm.router_utils.routing_groups import VALID_ROUTING_STRATEGIES
 from litellm.types.caching import LiteLLMCacheType
 from litellm.types.guardrails import GuardrailEventHooks, LitellmParams, SupportedGuardrailIntegrations
 from litellm.types.secret_managers.main import KeyManagementSystem

litellm/proxy/common_request_processing.py

@@ -53,6 +53,7 @@ from litellm.constants import (
 )
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.otel.runtime import phase_event
+from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
 from litellm.litellm_core_utils.bug_report import (
     allowlisted,
     bug_report_notice,
@@ -81,6 +82,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
 from litellm.litellm_core_utils.llm_response_utils.get_headers import (
     get_response_headers,
 )
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias, resolve_served_model
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.served_output_texts import (
     record_served_output_texts,
@@ -146,8 +148,6 @@ from litellm.responses.streaming_iterator import partial_responses_incomplete_ev
 from litellm.router import Router
 from litellm.router_api import RoutingContext
 from litellm.router_strategy.auto_router.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX
-from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict
-from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model
 from litellm.types.extraction import ExtractionResponse
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.router import RouterRateLimitError
@@ -2565,7 +2565,7 @@ class ProxyBaseLLMRequestProcessing:
         aliased_model: Final = self.data.get("model")
         if llm_router is not None and route_type in ROUTER_SCHEDULED_ROUTE_TYPES and isinstance(aliased_model, str):
             self.data.update(
-                llm_router.priority_header_update(
+                llm_router.health().priority_header_fields(
                     model=aliased_model, kwargs=self.data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
                 )
             )
@@ -2831,12 +2831,9 @@ class ProxyBaseLLMRequestProcessing:
 
     @staticmethod
     def _resolve_fallback_models(model: str, fallbacks: list) -> list[str] | None:
-        from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
+        from litellm.router_api import fallback_chain
 
-        fallback_model_group, generic_fallback_idx = get_fallback_model_group(
-            fallbacks=fallbacks,
-            model_group=model,
-        )
+        fallback_model_group, generic_fallback_idx = fallback_chain(fallbacks, model)
         if fallback_model_group is not None:
             return fallback_model_group
         return fallbacks[generic_fallback_idx]["*"] if generic_fallback_idx is not None else None
@@ -4190,8 +4187,8 @@ class ProxyBaseLLMRequestProcessing:
         HiddenParamsAsyncIteratorWrapper, so class sniffing runs on the
         unwrapped inner iterator.
         """
+        from litellm.litellm_core_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
         from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
-        from litellm.router_utils.add_retry_fallback_headers import HiddenParamsAsyncIteratorWrapper
 
         unwrapped: Final = response.inner if isinstance(response, HiddenParamsAsyncIteratorWrapper) else response
 
@@ -5034,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
         if stream_usage is None:
             return None
         service_tier: Final = obj.get("service_tier")
+        served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
         cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
-            ModelResponse(usage=stream_usage),
+            ModelResponse(model=served_model, usage=stream_usage),
             model_name,
             service_tier if isinstance(service_tier, str) else None,
             litellm_logging_obj,

litellm/proxy/common_utils/cache_aware_routing.py

@@ -16,6 +16,7 @@ from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_f
 from litellm.llms.anthropic.cache_aware_routing import AnthropicCacheRouting, TokenCounter
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
+from litellm.router_deprecated import renamed_router_param
 from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import Deployment, PreRoutingHookResponse
@@ -23,6 +24,7 @@ from litellm.types.utils import StandardLoggingRoutingDecision
 
 if TYPE_CHECKING:
     from litellm.router import Router
+    from litellm.router_api import RoutingContext
 
 _MESSAGES: Final = TypeAdapter(list[Mapping[str, object]])
 _MAPPING: Final = TypeAdapter(Mapping[str, object])
@@ -231,7 +233,7 @@ async def select_cached_model(
 
 async def _choose_cached_model(
     *,
-    router: Router,
+    ctx: RoutingContext,
     config: ComplexityRouterConfig,
     params_for_model: Callable[[str, str], Mapping[str, object]],
     response: PreRoutingHookResponse | None,
@@ -249,8 +251,10 @@ async def _choose_cached_model(
     )
     from litellm.router_strategy.complexity_router.context_compaction import compaction_pending
 
+    router: Final = proxy_server.llm_router
     if (
-        proxy_server.llm_router is not router
+        router is None
+        or not ctx.belongs_to(router)
         or router.registry().plugins("routing")
         or has_request_transforms()
         or compaction_pending(request_kwargs)
@@ -318,9 +322,10 @@ async def _choose_cached_model(
     )
 
 
+@renamed_router_param("router")
 async def choose_cached_model(
     *,
-    router: Router,
+    ctx: RoutingContext,
     config: ComplexityRouterConfig,
     params_for_model: Callable[[str, str], Mapping[str, object]],
     response: PreRoutingHookResponse | None,
@@ -332,7 +337,7 @@ async def choose_cached_model(
     try:
         return await asyncio.wait_for(
             _choose_cached_model(
-                router=router,
+                ctx=ctx,
                 config=config,
                 params_for_model=params_for_model,
                 response=response,

litellm/proxy/common_utils/credential_hydration.py

@@ -12,10 +12,10 @@ from types import MappingProxyType
 from typing import Final, cast  # noqa: TID251  # narrows untyped credential values at the decrypt boundary
 
 import litellm
+from litellm.litellm_core_utils.clientside_credential_handler import clientside_credential_keys
 from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.credentials_repository import CredentialsRepository
-from litellm.router_utils.clientside_credential_handler import clientside_credential_keys
 from litellm.types.router import (
     GenericLiteLLMParams,
     server_owned_wif_fields_present,

litellm/proxy/common_utils/model_listing_utils.py

@@ -21,7 +21,7 @@ from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm.constants import CLAUDE_CODE_CLIENT, CLAUDE_CODE_PICKER_PATTERN, GATEWAY_CLIENT_HEADER
-from litellm.router_utils.common_utils import resolve_model_group_alias
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 
 if TYPE_CHECKING:
     from litellm.router import Router

litellm/proxy/db/accounting_ledger.py

@@ -31,6 +31,13 @@ class AccountingConflictError(ValueError):
     pass
 
 
+@dataclass(frozen=True, slots=True)
+class UsageWrite:
+    event: InboundUsageEvent | OutboundUsageEvent
+    envelope: str
+    targets: tuple[DeliveryTarget, ...] = ()
+
+
 _Transaction: TypeAlias = AbstractAsyncContextManager["Prisma"]
 
 
@@ -107,6 +114,35 @@ SELECT event_id FROM "LiteLLM_AccountingEvent" WHERE kind = 'cost' AND source_ev
 _LOCK_EVENT: Final = """
 SELECT payload_hash FROM "LiteLLM_AccountingEvent" WHERE event_id = $1 FOR UPDATE
 """
+_INSERT_USAGE_BATCH: Final = """
+INSERT INTO "LiteLLM_AccountingEvent"
+    (event_id, kind, schema_version, payload_hash, occurred_at, payload)
+SELECT event_id, kind::"AccountingEventKind", 1, payload_hash, occurred_at::timestamp, payload
+FROM jsonb_to_recordset($1::jsonb) AS records(
+    event_id text, kind text, payload_hash text, occurred_at text, payload jsonb)
+ORDER BY event_id
+ON CONFLICT (event_id) DO NOTHING RETURNING event_id
+"""
+_LOCK_USAGE_BATCH: Final = """
+SELECT event_id, payload_hash FROM "LiteLLM_AccountingEvent"
+WHERE event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
+ORDER BY event_id FOR UPDATE
+"""
+_LOAD_USAGE_BATCH: Final = """
+SELECT event.payload, ARRAY(
+    SELECT target::text FROM "LiteLLM_AccountingDelivery" delivery
+    WHERE delivery.event_id = event.event_id ORDER BY target::text
+) AS targets
+FROM "LiteLLM_AccountingEvent" event
+WHERE event.event_id IN (SELECT value FROM jsonb_array_elements_text($1::jsonb))
+    AND event.kind = 'outbound_usage' AND event.schema_version = 1
+"""
+_INSERT_USAGE_DELIVERIES: Final = """
+INSERT INTO "LiteLLM_AccountingDelivery" (event_id, target, version)
+SELECT records.event_id, destination.value::"AccountingDeliveryTarget", 1
+FROM jsonb_to_recordset($1::jsonb) AS records(event_id text, targets jsonb)
+CROSS JOIN LATERAL jsonb_array_elements_text(records.targets) AS destination
+"""
 _COMPLETED_FAMILY: Final = """
 root.source_event_id IS NULL AND root.kind IN ('inbound_usage', 'outbound_usage')
 AND root.recorded_at < $1::timestamp
@@ -199,6 +235,53 @@ class AccountingLedger:
 
         return cls(transaction)
 
+    async def insert_usage_batch(self, writes: tuple[UsageWrite, ...]) -> None:
+        records: Final = tuple(_prepare(write.event, write.targets, write.envelope, 1, 1) for write in writes)
+        if not records:
+            return
+        unique: Final = {record.event_id: record for record in records}
+        if any(record.payload_hash != unique[record.event_id].payload_hash for record in records):
+            raise AccountingConflictError("Batch contains conflicting copies of a usage identity")
+        batch: Final = _json(
+            tuple(
+                {
+                    "event_id": record.event_id,
+                    "kind": record.kind.value,
+                    "payload_hash": record.payload_hash,
+                    "occurred_at": record.occurred_at.astimezone(timezone.utc).isoformat(),
+                    "payload": _JSON.validate_json(record.payload),
+                }
+                for record in unique.values()
+            )
+        )
+        async with self._transaction() as db:
+            inserted: Final = _STRINGS.validate_python(await db.query_raw(_INSERT_USAGE_BATCH, batch))
+            accepted: Final = _STRINGS.validate_python(await db.query_raw(_LOCK_USAGE_BATCH, _json(tuple(unique))))
+            if len(accepted) != len(unique) or any(
+                row["payload_hash"] != unique[row["event_id"]].payload_hash for row in accepted
+            ):
+                raise AccountingConflictError("Usage identity conflicts with accepted content or delivery set")
+            deliveries: Final = _json(
+                tuple({"event_id": row["event_id"], "targets": unique[row["event_id"]].targets} for row in inserted)
+            )
+            await db.execute_raw(_INSERT_USAGE_DELIVERIES, deliveries)
+
+    async def load_usage_batch(self, event_ids: tuple[str, ...]) -> tuple[UsageWrite, ...]:
+        if not event_ids:
+            return ()
+        async with self._transaction() as db:
+            rows: Final = TypeAdapter(tuple[dict[str, JsonValue], ...]).validate_python(
+                await db.query_raw(_LOAD_USAGE_BATCH, _json(event_ids))
+            )
+        return tuple(
+            UsageWrite(
+                OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
+                _json(row["payload"]),
+                TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
+            )
+            for row in rows
+        )
+
     async def insert_event(
         self,
         event: AccountingEvent,

litellm/proxy/db/exception_handler.py

@@ -72,6 +72,22 @@ def _exception_types(*candidates: object) -> tuple[type[BaseException], ...]:
     return tuple(c for c in candidates if isinstance(c, type) and issubclass(c, BaseException))
 
 
+def _prisma_error_code(e: Exception) -> str | None:
+    import prisma.engine.errors
+
+    if isinstance(e, _exception_types(prisma.errors.DataError)):
+        code: Final[object] = getattr(e, "code", None)
+        return code if isinstance(code, str) else None
+    if not isinstance(e, _exception_types(prisma.engine.errors.EngineRequestError)):
+        return None
+    try:
+        body: Final = _DATABASE_ERROR_META.validate_json(str(e).partition(": ")[2])
+    except ValidationError:
+        return None
+    engine_code: Final = body.get("error_code")
+    return engine_code if isinstance(engine_code, str) else None
+
+
 class PrismaDBExceptionHandler:
     """
     Class to handle DB Exceptions or Connection Errors
@@ -206,6 +222,8 @@ class PrismaDBExceptionHandler:
             return True
         if PrismaDBExceptionHandler.is_database_capacity_error(e):
             return False
+        if _prisma_error_code(e) == "P1017":
+            return True
         if isinstance(e, _exception_types(prisma.errors.PrismaError)):
             error_message: Final = str(e).lower()
             connection_keywords: Final = (

litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py

@@ -29,6 +29,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     has_tool_with_name,
 )
 from litellm.litellm_core_utils.provider_affinity import get_caller_session_id
+from litellm.litellm_core_utils.routing_common_utils import request_user_api_key_auth
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]
     httpxSpecialProvider,
@@ -40,7 +41,6 @@ from litellm.proxy.guardrails.guardrail_hooks.content_text import (
     merge_rewritten_text_parts,
 )
 from litellm.proxy.spend_tracking.compression_savings import HEADROOM_GUARDRAIL_PROVIDER
-from litellm.router_utils.common_utils import request_user_api_key_auth
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.guardrails import GuardrailEventHooks, Mode
 from litellm.types.integrations.custom_logger import (

litellm/proxy/guardrails/guardrail_hooks/semantic_guard/route_loader.py

@@ -14,6 +14,7 @@ from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_SEMANTIC_GUARD_SIMILARITY_THRESHOLD
+from litellm.router_api import RoutingContext
 
 if TYPE_CHECKING:
     from semantic_router.routers import SemanticRouter
@@ -134,7 +135,7 @@ class SemanticGuardRouteLoader:
         )
 
         encoder: Final = LiteLLMRouterEncoder(
-            litellm_router_instance=litellm_router,
+            ctx=RoutingContext(litellm_router),
             model_name=embedding_model,
             score_threshold=global_threshold,
         )

litellm/proxy/health_check.py

@@ -27,12 +27,12 @@ from litellm.constants import (
     HEALTH_CHECK_TIMEOUT_SECONDS,
 )
 from litellm.litellm_core_utils.health_check_helpers import native_health_check_mode
+from litellm.litellm_core_utils.routing_common_utils import deployment_model_id_pin, is_model_capability_field
 from litellm.router_strategy.auto_router.auto_router_model_naming import (
     StrategyRouterDependency,
     classify_strategy_router_model,
     strategy_router_dependencies,
 )
-from litellm.router_utils.common_utils import deployment_model_id_pin, is_model_capability_field
 from litellm.types.utils import secret_bearing_wif_litellm_params, server_owned_wif_litellm_params
 
 # Provider routing and server-owned federation or OAuth token-exchange fields. Allowed for proxy

litellm/proxy/health_check_utils/shared_health_check_manager.py

@@ -13,10 +13,10 @@ from litellm.caching.redis_cache import RedisCache
 from litellm.constants import (
     DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL,
     DEFAULT_SHARED_HEALTH_CHECK_TTL,
+    HEALTH_CHECKS_TARGET,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy.health_check import perform_health_check
-from litellm.router_utils.health_state_cache import HEALTH_CHECKS_TARGET
 
 if TYPE_CHECKING:
     from litellm.router import Router

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -26,7 +26,12 @@ from litellm.integrations.SlackAlerting.ms_teams import (
     build_ms_teams_payload,
     get_ms_teams_webhook_url,
 )
+from litellm.litellm_core_utils.clientside_credential_handler import (
+    _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE,  # pyright: ignore[reportPrivateUsage]  # one canonical list, shared with the router path
+    clientside_credential_keys,
+)
 from litellm.litellm_core_utils.custom_logger_registry import CustomLoggerRegistry
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.proxy._types import (
     AlertType,
@@ -94,11 +99,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
 from litellm.proxy.shutdown.graceful_shutdown_manager import GracefulShutdownManager
 from litellm.router import Router
 from litellm.router_api import deployment_usable_by_team
-from litellm.router_utils.clientside_credential_handler import (
-    _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE,  # pyright: ignore[reportPrivateUsage]  # one canonical list, shared with the router path
-    clientside_credential_keys,
-)
-from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.secret_managers.main import get_secret_bool
 from litellm.types.proxy.litellm_pre_call_utils import RedactedDict, SecretFields
 from litellm.types.router import reject_server_owned_wif_fields

litellm/proxy/hooks/dynamic_rate_limiter_v3.py

@@ -15,6 +15,10 @@ from litellm._internal_context import with_service_target
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.add_retry_fallback_headers import (
+    ensure_response_additional_headers,
+    response_has_hidden_params,
+)
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.common_utils.proxy_rate_limit_error import (
     ProxyRateLimitError,
@@ -34,10 +38,6 @@ from litellm.proxy.hooks.rate_limiter_utils import (
     resolve_llm_provider_for_rate_limit,
 )
 from litellm.proxy.utils import InternalUsageCache
-from litellm.router_utils.add_retry_fallback_headers import (
-    ensure_response_additional_headers,
-    response_has_hidden_params,
-)
 from litellm.types.router import ModelGroupInfo
 from litellm.types.utils import CallTypesLiteral, LLMResponseTypes
 

litellm/proxy/hooks/model_max_budget_limiter.py

@@ -204,7 +204,7 @@ def _team_public_names_by_group(router: "Router", team_id: str | None) -> Mappin
     groups_by_name: Final = MappingProxyType(
         {
             name: model_access_groups_of(router=router, model=name, team_id=team_id)
-            for name in router.team_public_model_names_of(team_id)
+            for name in router.registry().names(kind="team_public", team_id=team_id)
         }
     )
     return MappingProxyType(

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -49,6 +49,10 @@ from litellm.constants import (
 )
 from litellm.cost_calculator import get_usage_object
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.add_retry_fallback_headers import (
+    ensure_response_additional_headers,
+    response_has_hidden_params,
+)
 from litellm.litellm_core_utils.audio_utils.utils import estimate_audio_tokens
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
@@ -56,6 +60,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     split_media_for_tpm,
 )
 from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
+from litellm.litellm_core_utils.ptu_shares import PTUTeamCeiling
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 from litellm.litellm_core_utils.token_counter import estimate_image_tokens, offload_token_count
 from litellm.llms.azure.ptu_capacity import normalized_tokens
 from litellm.proxy._types import UserAPIKeyAuth
@@ -86,12 +92,6 @@ from litellm.proxy.spend_tracking.input_tokens import (
     openai_chat_pdf_file_tokens,
 )
 from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
-from litellm.router_utils.add_retry_fallback_headers import (
-    ensure_response_additional_headers,
-    response_has_hidden_params,
-)
-from litellm.router_utils.common_utils import resolve_model_group_alias
-from litellm.router_utils.ptu_shares import PTUTeamCeiling
 from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.extraction import ExtractionResponse
 from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
@@ -3561,9 +3561,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         Returns True if any deployment has failures in the current minute.
         """
         from litellm.proxy.proxy_server import llm_router
-        from litellm.router_utils.router_callbacks.track_deployment_metrics import (
-            get_deployment_failures_for_current_minute,
-        )
 
         if llm_router is None:
             return False
@@ -3581,10 +3578,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
                     continue
 
                 # Use router's existing failure tracking
-                failure_count = get_deployment_failures_for_current_minute(
-                    litellm_router_instance=llm_router,
-                    deployment_id=deployment_id,
-                )
+                failure_count = llm_router.health().failures_this_minute(deployment_id)
 
                 if failure_count > DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE:
                     verbose_proxy_logger.debug(

litellm/proxy/litellm_pre_call_utils.py

@@ -3291,7 +3291,7 @@ def _fallback_edges(
     and its provider-stripped suffix, plus the router matcher run over the fuzzy
     entries that have not fired yet (tracked in ``fired_fuzzy`` by
     ``(list_index, entry_index)``), so each entry's targets are expanded once."""
-    from litellm.router_utils.fallback_event_handlers import get_fallback_model_group
+    from litellm.router_api import fallback_chain
 
     stripped: Final = model_group.split("/", 1)[1] if "/" in model_group else None
     targets: Final[set[str]] = set()  # mutable-ok: accumulates fallback targets
@@ -3302,7 +3302,7 @@ def _fallback_edges(
         pending = tuple(index for index in range(len(fuzzy)) if (list_index, index) not in fired_fuzzy)
         if not pending:
             continue
-        resolved = get_fallback_model_group(fallbacks=[fuzzy[index] for index in pending], model_group=model_group)[0]
+        resolved = fallback_chain([fuzzy[index] for index in pending], model_group)[0]
         values = resolved if isinstance(resolved, list) else ([resolved] if resolved is not None else [])
         targets.update(name for value in values if (name := _fallback_target_name(value)) is not None)
         if resolved is not None:
@@ -3489,7 +3489,7 @@ async def _resolve_user_provider_credentials_for_request(
             status_code=400,
             detail=(f"fallback lists in the request body cannot exceed {_FALLBACK_DISCOVERY_LIMIT} entries"),
         )
-    from litellm.router_utils.common_utils import resolve_model_group_alias
+    from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 
     # Discovery must see the group the router actually routes to, so include the
     # post-alias name from every alias map that may still rewrite data["model"]

litellm/proxy/management_endpoints/auto_router_endpoints.py

@@ -60,6 +60,7 @@ from litellm.repositories.table_repositories import PrismaTableRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.repositories.verification_token_repository import VerificationTokenRepository
+from litellm.router_api import RoutingContext
 from litellm.router_strategy.auto_router.auto_router_model_naming import (
     StrategyRouterDependencyRole,
     classify_strategy_router_model,
@@ -614,7 +615,7 @@ async def preview_auto_router_routing(
     try:
         complexity_strategy: Final = ComplexityRouter(
             model_name=resolved.router_name,
-            litellm_router_instance=llm_router,
+            ctx=RoutingContext(llm_router),
             complexity_router_config=resolved.complexity_router_config.model_dump(exclude_none=True),
             default_model=resolved.complexity_router_config.resolve_default_model(resolved.default_model),
             derive_savings_baseline=False,

litellm/proxy/management_endpoints/ptu_consumption.py

@@ -12,14 +12,14 @@ from types import MappingProxyType
 from typing import Final
 
 from litellm.litellm_core_utils.ptu_pricing import is_ptu_cost_attribution_enabled
-from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
-from litellm.router import Router
-from litellm.router_utils.common_utils import resolve_model_group_alias
-from litellm.router_utils.ptu_shares import (
+from litellm.litellm_core_utils.ptu_shares import (
     model_group_ptu_capacity,
     routed_deployments,
     team_servable_deployments,
 )
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
+from litellm.llms.azure.ptu_capacity import PTUCapacity, normalized_tokens, ptu_hours
+from litellm.router import Router
 from litellm.types.proxy.management_endpoints.common_daily_activity import (
     DailySpendData,
     MetricWithMetadata,

litellm/proxy/management_helpers/model_quota_policy.py

@@ -9,10 +9,10 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator
 import litellm
 from litellm.exceptions import BadRequestError
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider, is_openai_compatible_endpoint
+from litellm.litellm_core_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
+from litellm.litellm_core_utils.routing_common_utils import provider_for_generic_call
 from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
-from litellm.router_utils.common_utils import provider_for_generic_call
-from litellm.router_utils.pattern_match_deployments import PatternMatchRouter, PatternUtils
 from litellm.secret_managers.main import get_secret_str
 
 if TYPE_CHECKING:

litellm/proxy/management_helpers/ptu_share_guard.py

@@ -5,8 +5,8 @@ from typing import Final
 
 from fastapi import HTTPException
 
+from litellm.litellm_core_utils.ptu_shares import deployments_shared_with
 from litellm.router import Router
-from litellm.router_utils.ptu_shares import deployments_shared_with
 
 
 def raise_if_teams_hold_ptu_shares(team_ids: Sequence[str], llm_router: Router | None) -> None:

litellm/proxy/mcp_server/auth/user_api_key_auth_mcp.py

@@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo
 _is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject
 
 
+def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
+    return (
+        user_api_key_auth is not None
+        and user_api_key_has_admin_view(user_api_key_auth)
+        and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID)
+    )
+
+
 def _gateway_dcr_challenge_target(
     route: str,
     mcp_servers: list[str] | None,
@@ -4290,19 +4298,19 @@ class MCPRequestHandler:
         """Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
         credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
 
-        Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
-        ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
-        admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
-        [] by DB default, so for that shape the row binds through the entitlement ceiling instead
-        (any non-empty entitlement, or an unresolved one, disqualifies), exactly as
-        ``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
-        server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
-        channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot
-        disagree."""
+        A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team
+        bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's
+        explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
+        admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB
+        default, so for that shape the row binds through the entitlement ceiling instead (any non-empty
+        entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same
+        row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers``
+        and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes
+        cannot disagree."""
         if (
             user_api_key_auth is None
             or user_api_key_auth.mcp_explicit_grants_only
-            or not user_api_key_has_admin_view(user_api_key_auth)
+            or not has_team_unbound_mcp_admin_view(user_api_key_auth)
         ):
             return False
         object_permission: Final = user_api_key_auth.object_permission

litellm/proxy/mcp_server/mcp_server_manager.py

@@ -3705,7 +3705,7 @@ class MCPServerManager:
 
         Priority:
         1. If object_permission.mcp_servers is explicitly set, use it (even for admins)
-        2. If admin and no object_permission, return all servers
+        2. If admin, not bound to a real team, and no object_permission, return all servers
         3. Otherwise, use standard permission checks
         """
         if managed_agent_policy(user_api_key_auth) is not None:

litellm/proxy/mcp_server/operations.py

@@ -17,6 +17,7 @@ import anyio
 from fastapi import HTTPException
 from mcp import MCPError, ReadResourceResult, Resource
 from mcp.types import (
+    METHOD_NOT_FOUND,
     CallToolRequest,
     CallToolRequestParams,
     CallToolResult,
@@ -222,6 +223,7 @@ __all__ = (
     "mcp_get_prompt",
     "mcp_read_resource",
     "raise_denied_scoped_mcp_access",
+    "raise_generic_mcp_access_denial",
 )
 
 
@@ -762,8 +764,14 @@ async def raise_denied_scoped_mcp_access(
                 )
             }
             raise HTTPException(status_code=403, detail=group_denial)
+    raise_generic_mcp_access_denial(requested_names)
+
+
+def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn:
     generic_denial: Final[_McpDeniedDetail] = {
-        "error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
+        "error": (
+            f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}"
+        )
     }
     raise HTTPException(status_code=403, detail=generic_denial)
 
@@ -2806,6 +2814,58 @@ async def _fire_mcp_tool_call_logging(
     return result
 
 
+async def log_mcp_sdk_rejection(
+    *,
+    method: str,
+    request_id: str | int | None,
+    error_code: int,
+    start_time: datetime,
+    user_api_key_auth: UserAPIKeyAuth,
+) -> None:
+    from litellm.proxy.proxy_server import proxy_logging_obj
+    from litellm.utils import Rules, function_setup
+
+    message: Final = "Method not found" if error_code == METHOD_NOT_FOUND else "Invalid request parameters"
+    rejection: Final = {
+        "method": "".join(char for char in method[:256] if char.isprintable()),
+        "request_id": (
+            "".join(char for char in str(request_id)[:128] if char.isprintable()) if request_id is not None else None
+        ),
+        "error_code": error_code,
+        "error_message": message,
+        "stage": "sdk_dispatch",
+        "route": user_api_key_auth.request_route or "/mcp",
+    }
+    metadata: Final = {
+        **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_auth),
+        "spend_logs_metadata": {"mcp_sdk_rejection": rejection},
+    }
+    logging_obj, request_data = function_setup(
+        original_function=CallTypes.call_mcp_tool.value,
+        rules_obj=Rules(),
+        start_time=start_time,
+        model="MCP: request_rejected",
+        litellm_call_id=str(uuid.uuid4()),
+        metadata=metadata,
+    )
+    logging_obj.update_environment_variables(  # pyright: ignore[reportUnknownMemberType]  # untyped legacy signature
+        litellm_params={"metadata": metadata},
+        optional_params={},
+        user=user_api_key_auth.user_id,
+    )
+    safe_error: Final = HTTPException(status_code=400, detail=message)
+    end_time: Final = datetime.now()  # noqa: DTZ005  # matches the failure logging start time
+    await asyncio.to_thread(logging_obj.failure_handler, safe_error, "", start_time, end_time)
+    await logging_obj.async_failure_handler(safe_error, "", start_time, end_time)
+    await proxy_logging_obj.post_call_failure_hook(  # pyright: ignore[reportUnknownMemberType]  # legacy signature
+        request_data={**request_data, "litellm_logging_obj": logging_obj},
+        original_exception=safe_error,
+        user_api_key_dict=user_api_key_auth,
+        route="/mcp",
+        traceback_str="",
+    )
+
+
 async def fire_mcp_tool_call_failure_logging(
     logging_obj: LiteLLMLoggingObj | None,
     exception: Exception,

litellm/proxy/mcp_server/semantic_tool_filter.py

@@ -14,6 +14,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import ExceptionCheckers
 from litellm.proxy.mcp_server.catalog import catalog_operation, global_manager
 from litellm.proxy.mcp_server.faults import iter_exception_tree
 from litellm.proxy.mcp_server.utils import MCP_TOOL_PREFIX_SEPARATOR
+from litellm.router_api import RoutingContext
 
 if TYPE_CHECKING:
     from semantic_router.routers import SemanticRouter
@@ -172,7 +173,7 @@ class SemanticMCPToolFilter:
             self.tool_router = SemanticRouter(
                 routes=routes,
                 encoder=LiteLLMRouterEncoder(
-                    litellm_router_instance=self.router_instance,
+                    ctx=RoutingContext(self.router_instance),
                     model_name=self.embedding_model,
                     score_threshold=self.similarity_threshold,
                 ),
@@ -247,7 +248,7 @@ class SemanticMCPToolFilter:
                 router: Final = SemanticRouter(
                     routes=[],
                     encoder=LiteLLMRouterEncoder(
-                        litellm_router_instance=self.router_instance,
+                        ctx=RoutingContext(self.router_instance),
                         model_name=self.embedding_model,
                         score_threshold=self.similarity_threshold,
                     ),

litellm/proxy/mcp_server/server.py

@@ -14,6 +14,7 @@ import time
 import types
 from collections import Counter
 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, Sequence, Set
+from datetime import datetime
 from typing import TYPE_CHECKING, Final, Literal, NoReturn, Protocol
 
 import httpx
@@ -519,6 +520,7 @@ if MCP_AVAILABLE:
         "mcp_get_prompt",
         "mcp_read_resource",
         "raise_denied_scoped_mcp_access",
+        "raise_generic_mcp_access_denial",
         "redact_mcp_resource_url",
     )
     from mcp.server import Server
@@ -530,13 +532,15 @@ if MCP_AVAILABLE:
     )
     from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser
     from mcp.server.auth.provider import AccessToken
-    from mcp.server.context import ServerRequestContext
+    from mcp.server.context import CallNext, HandlerResult, ServerRequestContext
     from mcp.server.lowlevel.server import NotificationOptions
     from mcp.server.lowlevel.server import Server as MCPProtocolServer
     from mcp.server.models import InitializationOptions
     from mcp.shared.exceptions import MCPError
     from mcp.shared.inbound import InboundLadderRejection, classify_inbound_request, find_duplicated_routing_header
     from mcp.types import (
+        INVALID_PARAMS,
+        METHOD_NOT_FOUND,
         CallToolRequest,
         GetPromptRequest,
         JSONRPCRequest,
@@ -546,6 +550,7 @@ if MCP_AVAILABLE:
         ListToolsRequest,
         ReadResourceRequest,
     )
+    from mcp_types.methods import CLIENT_REQUESTS, SPEC_CLIENT_METHODS
 
     from litellm.proxy.mcp_server import operations
     from litellm.proxy.mcp_server.contracts import OperationContext
@@ -643,7 +648,56 @@ if MCP_AVAILABLE:
         name=LITELLM_MCP_SERVER_NAME,
         version=LITELLM_MCP_SERVER_VERSION,
     )
-    server.middleware.extend((finish_scope_response, GatewayVersionPolicy()))
+
+    def _sdk_rejection_code(ctx: ServerRequestContext, error: MCPError | ValidationError) -> int | None:
+        if ctx.request_id is None:
+            return None
+        entry: Final = server.get_request_handler(ctx.method)
+        if entry is None:
+            if ctx.method == "initialize":
+                return None
+            if isinstance(error, ValidationError):
+                return INVALID_PARAMS
+            if error.code == METHOD_NOT_FOUND:
+                return METHOD_NOT_FOUND
+            return None
+        if not isinstance(error, ValidationError):
+            unavailable_for_version: Final = (
+                ctx.method in SPEC_CLIENT_METHODS and (ctx.method, ctx.protocol_version) not in CLIENT_REQUESTS
+            )
+            return METHOD_NOT_FOUND if error.code == METHOD_NOT_FOUND and unavailable_for_version else None
+        try:
+            entry.params_type.model_validate({} if ctx.params is None else ctx.params, by_name=False)
+        except ValidationError:
+            return INVALID_PARAMS
+        return None
+
+    async def _log_sdk_rejection(ctx: ServerRequestContext, call_next: CallNext) -> HandlerResult:
+        start_time: Final = datetime.now()  # noqa: DTZ005  # failure logging uses naive timestamps
+        try:
+            return await call_next(ctx)
+        except (MCPError, ValidationError) as error:
+            try:
+                code: Final = _sdk_rejection_code(ctx, error)
+                if code is not None:
+                    token: Final = active_mcp_request_ctx_var.set(ctx)
+                    try:
+                        auth, *_ = await get_or_extract_auth_context()
+                    finally:
+                        active_mcp_request_ctx_var.reset(token)
+                    if auth is not None:
+                        await operations.log_mcp_sdk_rejection(
+                            method=ctx.method,
+                            request_id=ctx.request_id,
+                            error_code=code,
+                            start_time=start_time,
+                            user_api_key_auth=auth,
+                        )
+            except Exception:  # noqa: BLE001  # logging must not replace the SDK response or expose rejected input
+                verbose_logger.warning("Failed to log MCP SDK rejection")
+            raise
+
+    server.middleware.extend((finish_scope_response, GatewayVersionPolicy(), _log_sdk_rejection))
     server.create_initialization_options = types.MethodType(_gateway_create_initialization_options, server)
     sse: Final[SseServerTransport] = SseServerTransport("/sse/messages")
 
@@ -1144,6 +1198,7 @@ if MCP_AVAILABLE:
         apply_display_name_overrides,
         filter_tools_by_allowed_tools,
         raise_denied_scoped_mcp_access,
+        raise_generic_mcp_access_denial,
     )
 
     @contextlib.asynccontextmanager
@@ -1771,10 +1826,10 @@ if MCP_AVAILABLE:
         row capping it.
 
         Raises HTTPException(403) unless the principal holds toolset_id through one
-        of its grant sources. Admins always pass.
+        of its grant sources. Admins pass unless the key is bound to a real team.
         """
         from litellm.proxy._types import LiteLLM_ObjectPermissionTable
-        from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view
+        from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view
 
         # A key scoped to no MCP servers opts out of every MCP path. Enforce it
         # here too, since toolset scoping replaces mcp_servers and would otherwise
@@ -1788,7 +1843,7 @@ if MCP_AVAILABLE:
             )
 
         acting: Final = await acting_user(user_api_key_auth)
-        is_admin: Final = user_api_key_has_admin_view(acting)
+        is_admin: Final = has_team_unbound_mcp_admin_view(acting)
         if not is_admin and toolset_id not in await granted(acting):
             raise HTTPException(
                 status_code=403,

litellm/proxy/openai_files_endpoints/files_endpoints.py

@@ -1786,7 +1786,7 @@ def _routed_model_name_on_provider(
 def _stored_model_name_routes_to_provider(
     llm_router: Router, custom_llm_provider: str, provider_model_names: frozenset[str], model_name: str
 ) -> bool:
-    routing_group: Final = llm_router.get_routing_group(model_name)
+    routing_group: Final = llm_router.registry().routing_group(model_name)
     routed_model_names: Final = (
         tuple(routing_group.models) if routing_group is not None else (llm_router.registry().resolve(model_name),)
     )

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
 from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
 from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
-from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
+from litellm.llms.anthropic.common_utils import (
+    AnthropicModelInfo,
+    anthropic_pass_through_api_base,
+    merge_anthropic_beta_headers,
+)
 from litellm.llms.azure.passthrough.transformation import (
     foreign_azure_deployment,
     is_azure_body_model_inference_endpoint,
@@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
     """
     [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
     """
-    base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
+    base_target_url = anthropic_pass_through_api_base()
     encoded_endpoint = httpx.URL(endpoint).path
 
     # Ensure endpoint starts with '/' for proper URL construction

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
+from litellm.llms.anthropic.common_utils import (
+    anthropic_accepts_user_id,
+    anthropic_upstream_user_id_limit,
+    is_anthropic_messages_url,
+)
 from litellm.llms.base_llm.managed_resources.utils import (
     resolve_passthrough_managed_id_provider,
 )
@@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
     EndpointType,
     PassthroughStandardLoggingPayload,
+    WebSocketSessionMeter,
+    WebSocketSessionUsage,
 )
 from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
 
@@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         url: httpx.URL,
         headers: dict,
         requested_query_params: dict | None = None,
-        _parsed_body: dict | None = None,
+        _parsed_body: Mapping[str, object] | None = None,
         forward_multipart: bool = False,
     ) -> httpx.Response:
         """
@@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         # unenforced. Set after the client merge so a request body cannot supply its own budget.
         #
         # Only for the built-in provider routes. `get_model_from_request` returns
-        # None for a user-defined pass-through, deliberately: its body is forwarded
-        # verbatim, so `model` there names an UPSTREAM model rather than a
+        # None for a user-defined pass-through, deliberately: the client's `model`
+        # goes upstream unchanged, so it names an UPSTREAM model rather than a
         # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
         # charging a counter anyway would track spend that nothing can refuse, and
         # would attribute it to a budget the operator scoped to a LiteLLM model that
@@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
     return headers
 
 
+def _json_object(value: object) -> Mapping[str, object]:
+    return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({})  # cast-ok: json
+
+
+def _anthropic_user_id_to_forward(
+    body: Mapping[str, object],
+    url: str,
+    has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
+) -> str | None:
+    user_id: Final = _json_object(body.get("metadata")).get("user_id")
+    if not isinstance(user_id, str):
+        return None
+    max_length: Final = anthropic_upstream_user_id_limit(url)
+    if max_length is None:
+        return None
+    if not anthropic_accepts_user_id(user_id, max_length):
+        verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
+        return None
+    if has_pre_call_guardrails(body):
+        verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
+        return None
+    return user_id
+
+
+def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
+    if body is None or user_id is None:
+        return body
+    return {**body, "metadata": {"user_id": user_id}}
+
+
 def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
     return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
 
@@ -1421,6 +1457,10 @@ async def pass_through_request(
             request_method=getattr(request, "method", None),
             cost_per_request=cost_per_request,
         )
+        hooked_body: Final = _json_object(_parsed_body)
+        client_user_id: Final = _anthropic_user_id_to_forward(
+            hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
+        )
         kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint(  # rebind-ok: pre-existing rebinding on a rename-only line
             user_api_key_dict=user_api_key_dict,
             _parsed_body=_parsed_body,
@@ -1588,11 +1628,13 @@ async def pass_through_request(
             else:
                 logging_url = str(url) + "?" + requested_query_params_str
 
+        outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body)  # cast-ok: json
+        upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
         logging_obj.pre_call(
             input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
             api_key="",
             additional_args={
-                "complete_input_dict": _parsed_body,
+                "complete_input_dict": upstream_body,
                 "api_base": str(logging_url),
                 "headers": redact_credential_headers(upstream_headers),
             },
@@ -1632,7 +1674,7 @@ async def pass_through_request(
                         url,
                         params=requested_query_params,
                         headers=upstream_headers,
-                        json=_parsed_body,
+                        json=upstream_body,
                     )
                 )
 
@@ -1731,7 +1773,7 @@ async def pass_through_request(
                 url=url,
                 headers=upstream_headers,
                 requested_query_params=requested_query_params,
-                _parsed_body=_parsed_body,
+                _parsed_body=upstream_body,
                 forward_multipart=is_multipart,
             )
         verbose_proxy_logger.debug("response.headers= %s", response.headers)
@@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
     return inject_trace_context(headers, parent_span=parent_span)
 
 
+def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
+    logging_obj.model = usage.model  # rebind-ok: the spend logger reads model and cost off the shared logging object
+    logging_obj.model_call_details["model"] = usage.model  # rebind-ok: same shared logging object
+    logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider  # rebind-ok: same
+    logging_obj.model_call_details["response_cost"] = usage.response_cost  # rebind-ok: same
+
+
 async def websocket_passthrough_request(
     websocket: WebSocket,
     target: str,
@@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
     setup_model_rewriter: Callable[[str], str] | None = None,
     builtin_provider_route: bool = False,
     model: str | None = None,
+    meter: WebSocketSessionMeter | None = None,
 ):
     """
     WebSocket passthrough request handler.
@@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
         setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
         builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
         model: The model the caller named before connecting, logged on a failure before the setup frame arrives
+        meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
     """
     from litellm.litellm_core_utils.litellm_logging import Logging
     from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
     # Initialize tracking variables
     start_time: Final = datetime.now()
     websocket_messages: Final[list[dict[str, object]]] = []
+    metered_total = 0  # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
     litellm_call_id: Final = str(uuid.uuid4())
 
     verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
@@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
 
             async def forward_client_to_upstream() -> None:
                 """Forward messages from client to upstream WebSocket"""
+                nonlocal metered_total
                 try:
                     while True:
                         message = await websocket.receive()
@@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
                                     # Not a JSON message or doesn't contain setup data
 
                             await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
+                            if meter is not None:
+                                metered_total += meter.measure(text_data)
                         elif bytes_data is not None:
                             await upstream_ws.send(bytes_data)
                 except asyncio.CancelledError:
@@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
                     self.url = target_url
 
             mock_response: Final = MockWebSocketResponse(target)
+            if meter is not None:
+                _log_session_usage(logging_obj, meter.price(metered_total))
 
             # Use the same success handler as HTTP passthrough endpoints
             GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

litellm/proxy/policy_engine/response_retrieval.py

@@ -7,6 +7,7 @@ from pydantic import TypeAdapter
 
 from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 from litellm.proxy.common_utils.callback_utils import (
     add_guardrail_to_applied_guardrails_header,
     add_policy_sources_to_metadata,
@@ -18,7 +19,6 @@ from litellm.proxy.policy_engine.policy_matcher import PolicyMatcher
 from litellm.proxy.policy_engine.policy_registry import get_policy_registry
 from litellm.proxy.policy_engine.policy_resolver import PolicyResolver
 from litellm.responses.utils import ResponsesAPIRequestUtils
-from litellm.router_utils.common_utils import resolve_model_group_alias
 from litellm.types.proxy.policy_engine import PolicyMatchContext
 from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
 

litellm/proxy/proxy_server.py

@@ -89,6 +89,11 @@ from litellm.constants import (
     OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS,
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
 )
+from litellm.litellm_core_utils.access_windows import access_windows_config_error
+from litellm.litellm_core_utils.add_retry_fallback_headers import (
+    get_fallback_errors_from_headers,
+    get_hidden_params_dict,
+)
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.bug_report import (
     allowlisted,
@@ -100,6 +105,8 @@ from litellm.litellm_core_utils.litellm_logging import (
     initialize_sentry,
 )
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
+from litellm.litellm_core_utils.routing_groups import parse_routing_groups
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.token_counter import (
@@ -189,13 +196,6 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
     snapshot_tuning_baselines,
     tuning_limit_violation,
 )
-from litellm.router_utils.access_windows import access_windows_config_error
-from litellm.router_utils.add_retry_fallback_headers import (
-    get_fallback_errors_from_headers,
-    get_hidden_params_dict,
-)
-from litellm.router_utils.common_utils import resolve_model_group_alias
-from litellm.router_utils.routing_groups import parse_routing_groups
 from litellm.types.caching import RedisPipelineIncrementOperation
 from litellm.types.utils import (
     PRICING_OVERRIDES_KEY,
@@ -1001,7 +1001,7 @@ from litellm.router import (
     LiteLLM_Params,
     ModelGroupInfo,
 )
-from litellm.router_api import ChangeSet, deployment_id
+from litellm.router_api import ChangeSet, deployment_id, router_init_args
 from litellm.scheduler import FlowItem, Scheduler
 from litellm.secret_managers.aws_secret_manager import load_aws_kms
 from litellm.secret_managers.google_kms import load_google_kms
@@ -4669,10 +4669,6 @@ def _write_health_state_to_router_cache(
     is on.
     """
     from litellm.proxy.health_check import build_deployment_health_states
-    from litellm.router_utils.cooldown_handlers import set_cooldown_deployments
-    from litellm.router_utils.router_callbacks.track_deployment_metrics import (
-        increment_deployment_failures_for_current_minute,
-    )
 
     _exceptions: Final[dict] = exceptions_by_model_id or {}
 
@@ -4724,18 +4720,7 @@ def _write_health_state_to_router_cache(
             ):
                 continue
 
-            increment_deployment_failures_for_current_minute(
-                litellm_router_instance=llm_router,
-                deployment_id=model_id,
-            )
-
-            set_cooldown_deployments(
-                litellm_router_instance=llm_router,
-                original_exception=original_exception,
-                exception_status=exception_status,
-                deployment=model_id,
-                time_to_cooldown=router_settings.cooldown_time,
-            )
+            llm_router.health().record_failure(model_id, original_exception, exception_status)
 
     except Exception as e:
         verbose_proxy_logger.warning("Failed to write health state to router cache: %s", str(e))
@@ -7379,9 +7364,7 @@ class ProxyConfig:
         router_settings: Final = config.get("router_settings", None)
 
         if router_settings and isinstance(router_settings, dict):
-            available_args: Final = [
-                x for x in litellm.Router.get_valid_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
-            ]
+            available_args: Final = [x for x in router_init_args() if x not in ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG]
 
             for k, v in router_settings.items():
                 if k in available_args:
@@ -9605,7 +9588,6 @@ class ProxyConfig:
             SearchToolRegistry,
             keep_loaded_search_tools_that_do_not_decrypt,
         )
-        from litellm.router_utils.search_api_router import SearchAPIRouter
 
         try:
             db_search_tools: Final = keep_loaded_search_tools_that_do_not_decrypt(
@@ -9629,7 +9611,21 @@ class ProxyConfig:
             )
 
             if llm_router is not None:
-                await SearchAPIRouter.update_router_search_tools(router_instance=llm_router, search_tools=search_tools)
+                llm_router.apply(
+                    ChangeSet(
+                        assign={
+                            "search_tools": [
+                                {
+                                    "search_tool_id": tool.get("search_tool_id"),
+                                    "search_tool_name": tool.get("search_tool_name"),
+                                    "litellm_params": tool.get("litellm_params", {}),
+                                    "search_tool_info": tool.get("search_tool_info"),
+                                }
+                                for tool in search_tools
+                            ]
+                        }
+                    )
+                )
                 verbose_proxy_logger.info("Successfully loaded %s search tool(s) into router", len(search_tools))
             else:
                 verbose_proxy_logger.debug(
@@ -16161,7 +16157,7 @@ def _enrich_model_info_with_litellm_data(
     if debug is True:
         _openai_client = "None"
         if llm_router is not None:
-            _openai_client = llm_router._get_client(deployment=model, kwargs={}, client_type="async") or "None"
+            _openai_client = llm_router.registry().client(model, client_type="async") or "None"
         else:
             _openai_client = "llm_router_is_None"
         openai_client: Final = str(_openai_client)
@@ -18611,7 +18607,7 @@ async def async_queue_request(
         router: Final = llm_router
 
         data.update(
-            router.priority_header_update(
+            router.health().priority_header_fields(
                 model=data["model"], kwargs=data, header=request.headers.get(SCHEDULER_PRIORITY_HEADER)
             )
         )
@@ -19760,7 +19756,7 @@ async def update_config(
         raw_router_settings: Final = request_body.get("router_settings")
         if isinstance(raw_router_settings, dict):
             supported_router_settings: Final = RUNTIME_UPDATABLE_ROUTER_SETTINGS | (
-                frozenset(litellm.Router.get_valid_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
+                frozenset(router_init_args()) - ROUTER_SETTINGS_MANAGED_OUTSIDE_CONFIG
             )
             unsupported_router_settings: Final = sorted(set(raw_router_settings) - supported_router_settings)
             if unsupported_router_settings:
@@ -21917,6 +21913,7 @@ async def _stream_mcp_asgi_response(handle_fn, scope: dict, receive) -> "Streami
             while True:
                 chunk = await body_queue.get()
                 if chunk is None:
+                    await handler_task
                     break
                 yield chunk
         finally:
@@ -21997,9 +21994,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
     Namespace a toolset as its own MCP endpoint.
 
     Connecting to /toolset/<name>/mcp exposes exactly the tools defined in
-    the toolset. Access is enforced: non-admin API keys must have the toolset
-    listed in their object_permission.mcp_toolsets grant list, or the request
-    will be rejected with a 403.
+    the toolset. Access is enforced: API keys other than an admin's key with no
+    team must hold the toolset through one of their grant sources, or the
+    request will be rejected with a 403.
     """
     try:
         from litellm.proxy.mcp_server.mcp_server_manager import (

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -3343,6 +3343,34 @@
     ],
     "default_model_placeholder": "prism/deepseek-v4.1-flash"
   },
+  {
+    "provider": "HOSTINGER_ROUTER",
+    "provider_display_name": "Hostinger Router",
+    "litellm_provider": "hostinger_router",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://router.hostinger.com/v1",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "hostinger_router/claude-sonnet-5"
+  },
   {
     "provider": "RECRAFT",
     "provider_display_name": "Recraft",

litellm/proxy/route_llm_request.py

@@ -16,8 +16,9 @@ from pydantic import BeforeValidator, TypeAdapter, ValidationError
 import litellm
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
-from litellm.router_utils.common_utils import is_proxy_admin_request
+from litellm.router_api import router_init_args
 
 # Client-supplied params that make the router or the call path fabricate a
 # failure or a delay instead of calling the provider. The ``mock_testing_*``
@@ -93,7 +94,7 @@ async def _route_user_config_request(data: dict, route_type: str):
 
     # Filter router_config to only include valid Router.__init__ arguments
     # This prevents TypeError when invalid parameters are stored in the database
-    valid_args: Final = litellm.Router.get_valid_args()
+    valid_args: Final = router_init_args()
     filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
 
     logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)

litellm/proxy/spend_tracking/input_tokens.py

@@ -21,6 +21,7 @@ from typing import TYPE_CHECKING, Final
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import OPENAI_CHAT_PDF_FILE_TOKENS, PDF_DATA_URL_PREFIX
+from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models, str_keyed
 from litellm.litellm_core_utils.token_counter import (
     deployment_openai_model,
     pdf_page_pricing,
@@ -31,7 +32,6 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
     billable_input_since_newest_compaction,
     holds_compaction_summary,
 )
-from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed
 from litellm.rust_bridge import runtime
 from litellm.rust_bridge.catalog import Route, RouteContext
 from litellm.rust_bridge.token_counter import (
@@ -162,7 +162,7 @@ def models_counted_for(model: str, llm_router: Router | None, team_id: str | Non
 def _routed_deployments(model: str, llm_router: Router, team_id: str | None) -> Sequence[object]:
     registry: Final = llm_router.registry()
     resolved: Final = registry.resolve(model, kind="alias") or model
-    routing_group: Final = llm_router.get_routing_group(resolved)
+    routing_group: Final = registry.routing_group(resolved)
     members: Final = routing_group.models if routing_group is not None else (resolved,)
     named: Final = tuple(
         chain.from_iterable(registry.deployments(member, source="all", team_id=team_id) for member in members)

litellm/proxy/spend_tracking/usage_meter.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import asyncio
+from collections import deque
+from collections.abc import Awaitable, Callable
+from concurrent.futures import Future
+from dataclasses import dataclass
+from itertools import accumulate, chain, count, takewhile
+from threading import Lock
+from typing import Final
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from litellm._logging import verbose_proxy_logger
+from litellm.accounting import (
+    DeliveryTarget,
+    Dimension,
+    DimensionName,
+    InboundUsageEvent,
+    OutboundUsageEvent,
+)
+from litellm.proxy.db.accounting_ledger import AccountingConflictError, UsageWrite
+
+_DAILY_TARGETS: Final = (
+    (DimensionName.KEY, DeliveryTarget.DAILY_USER_SPEND),
+    (DimensionName.USER, DeliveryTarget.DAILY_USER_SPEND),
+    (DimensionName.TEAM, DeliveryTarget.DAILY_TEAM_SPEND),
+    (DimensionName.ORGANIZATION, DeliveryTarget.DAILY_ORGANIZATION_SPEND),
+    (DimensionName.END_USER, DeliveryTarget.DAILY_END_USER_SPEND),
+    (DimensionName.AGENT, DeliveryTarget.DAILY_AGENT_SPEND),
+    (DimensionName.TAG, DeliveryTarget.DAILY_TAG_SPEND),
+    (DimensionName.MODEL, DeliveryTarget.DAILY_MODEL_USAGE),
+    (DimensionName.TOOL, DeliveryTarget.DAILY_TOOL_SPEND),
+)
+
+
+def usage_delivery_targets(dimensions: tuple[Dimension, ...]) -> tuple[DeliveryTarget, ...]:
+    names: Final = frozenset(dimension.name for dimension in dimensions)
+    return tuple(dict.fromkeys(target for dimension, target in _DAILY_TARGETS if dimension in names))
+
+
+class UsageCaptureSettings(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="forbid")
+
+    batch_size: int = Field(default=1000, ge=2, le=1000)
+    flush_interval_seconds: float = Field(default=5.0, gt=0, le=10)
+    max_pending_bytes: int = Field(default=16 * 1024 * 1024, ge=1024)
+    max_pending_records: int = Field(default=10000, ge=2)
+
+
+class UsageCaptureError(RuntimeError):
+    pass
+
+
+@dataclass(frozen=True, slots=True)
+class _Pending:
+    writes: tuple[UsageWrite, ...]
+    size: int
+    committed: Future[None]
+    finished: asyncio.Event
+    prerequisites: tuple[asyncio.Future[None], ...]
+
+
+async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None:
+    try:
+        await asyncio.wait_for(wakeup.wait(), timeout)
+    except asyncio.TimeoutError:
+        return
+
+
+def _report_failure(receipt: Future[None]) -> None:
+    if not receipt.cancelled() and receipt.exception() is not None:
+        verbose_proxy_logger.error("Usage capture did not acknowledge commit", exc_info=receipt.exception())
+
+
+def _retryable_persistence_failure(error: Exception) -> bool:
+    from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
+
+    outage: Final = PrismaDBExceptionHandler.find_database_service_unavailable_error_in_chain(error)
+    return (outage is not None and not PrismaDBExceptionHandler.is_permanent_database_fault(outage)) or any(
+        isinstance(cause, Exception)
+        and (
+            PrismaDBExceptionHandler.is_deadlock_error(cause)
+            or PrismaDBExceptionHandler.is_lock_timeout_error(cause)
+            or PrismaDBExceptionHandler.postgres_sqlstate(cause) == "40001"
+        )
+        for cause in exception_chain(error)
+    )
+
+
+def _isolatable_persistence_failure(error: Exception) -> bool:
+    from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
+
+    return isinstance(error, AccountingConflictError) or any(
+        isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22")
+        for cause in exception_chain(error)
+    )
+
+
+class DurableUsageMeter:
+    def __init__(
+        self,
+        write_batch: Callable[[tuple[UsageWrite, ...]], Awaitable[None]],
+        settings: UsageCaptureSettings,
+        *,
+        wait_for_batch: Callable[[asyncio.Event, float], Awaitable[None]] = _wait_for_batch,
+        retry_delay: Callable[[float], Awaitable[None]] = asyncio.sleep,
+    ) -> None:
+        self._write_batch: Final = write_batch
+        self.settings: Final = settings
+        self._wait_for_batch: Final = wait_for_batch
+        self._retry_delay: Final = retry_delay
+        self._loop: Final = asyncio.get_running_loop()
+        self._admission: Final = Lock()
+        self._queue: Final[deque[_Pending]] = deque()  # mutable-ok: one bounded producer/consumer queue
+        self._wakeup: Final = asyncio.Event()
+        self._task: asyncio.Task[None] | None = None
+        self._closed = False
+        self.pending_bytes = 0
+        self.pending_records = 0
+
+    def submit(
+        self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = ()
+    ) -> asyncio.Future[None]:
+        size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
+        with self._admission:
+            if (
+                self._closed
+                or not writes
+                or len(writes) > self.settings.batch_size
+                or self.pending_bytes + size > self.settings.max_pending_bytes
+                or self.pending_records + len(writes) > self.settings.max_pending_records
+            ):
+                raise UsageCaptureError("Usage capture admission rejected: closed, empty or capacity exhausted")
+            receipt: Final = Future[None]()
+            receipt.set_running_or_notify_cancel()
+            receipt.add_done_callback(_report_failure)
+            self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites))
+            self.pending_bytes += size
+            self.pending_records += len(writes)
+            self._loop.call_soon_threadsafe(self._ensure_worker)
+        wrapped: Final = asyncio.wrap_future(receipt, loop=self._loop)
+        wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
+        return wrapped
+
+    def _ensure_worker(self) -> None:
+        if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
+            self._wakeup.set()
+        if self._task is None or self._task.done():
+            self._task = asyncio.create_task(self._run(), name="accounting usage capture")
+
+    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
+        envelope: Final = '{"event":' + event.model_dump_json() + "}"
+        await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
+
+    async def flush(self) -> None:
+        self._ensure_worker()
+        self._wakeup.set()
+        if self._task is not None:
+            await asyncio.shield(self._task)
+
+    async def close(self) -> None:
+        with self._admission:
+            self._closed = True
+        await self.flush()
+
+    async def stop(self, timeout: float) -> None:
+        with self._admission:
+            self._closed = True
+        try:
+            await asyncio.wait_for(self.close(), timeout)
+        except asyncio.TimeoutError:
+            if self._task is not None:
+                self._task.cancel()
+                await asyncio.gather(self._task, return_exceptions=True)
+            while self._queue:
+                self._finish(
+                    self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
+                )
+            verbose_proxy_logger.error("Usage capture shutdown deadline exhausted; commit acknowledgement incomplete")
+
+    def _take_batch(self) -> tuple[_Pending, ...]:
+        with self._admission:
+            if self._queue[0].prerequisites:
+                return (self._queue.popleft(),)
+            independent: Final = takewhile(lambda item: not item.prerequisites, self._queue)
+            sizes: Final = accumulate(len(item.writes) for item in independent)
+            count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes))
+            return tuple(self._queue.popleft() for _ in range(count))
+
+    async def _persist(self, writes: tuple[UsageWrite, ...]) -> None:
+        for attempt in count():
+            try:
+                await self._write_batch(writes)
+            except Exception as error:  # noqa: BLE001  # retry only classified database failures, preserving frozen writes
+                if not _retryable_persistence_failure(error):
+                    raise
+                verbose_proxy_logger.warning("Usage persistence unavailable; retaining admitted batch for retry")
+                await self._retry_delay(min(10.0, 0.2 * 2 ** min(attempt, 6)))
+            else:
+                return
+
+    async def _commit(self, batch: tuple[_Pending, ...]) -> None:
+        writes: Final = tuple(chain.from_iterable(item.writes for item in batch))
+        try:
+            if batch[0].prerequisites:
+                outcomes: Final = await asyncio.gather(
+                    *(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True
+                )
+                for outcome in outcomes:
+                    if isinstance(outcome, BaseException):
+                        raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome
+            await self._persist(writes)
+        except asyncio.CancelledError:
+            self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
+            raise
+        except Exception as error:  # noqa: BLE001  # isolate data faults, surface systemic failures to all producers
+            if len(batch) > 1 and _isolatable_persistence_failure(error):
+                try:
+                    for item in batch:
+                        await self._commit((item,))
+                except asyncio.CancelledError:
+                    self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
+                    raise
+                return
+            self._finish(
+                batch,
+                UsageCaptureError("Usage identity conflicts with accepted content")
+                if isinstance(error, AccountingConflictError)
+                else error,
+            )
+        else:
+            self._finish(batch, None)
+
+    def _finish(self, batch: tuple[_Pending, ...], error: Exception | None) -> None:
+        for item in batch:
+            if item.finished.is_set():
+                continue
+            item.finished.set()
+            with self._admission:
+                self.pending_bytes -= item.size
+                self.pending_records -= len(item.writes)
+            if item.committed.done():
+                continue
+            if error is None:
+                item.committed.set_result(None)
+            else:
+                item.committed.set_exception(error)
+
+    async def _run(self) -> None:
+        try:
+            while self._queue:
+                await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds)
+                self._wakeup.clear()
+                await self._commit(self._take_batch())
+                if (
+                    self._closed
+                    or self.pending_records >= self.settings.batch_size
+                    or self._queue
+                    and self._queue[0].prerequisites
+                ):
+                    self._wakeup.set()
+        except (Exception, asyncio.CancelledError):  # noqa: BLE001  # reject queued receipts if the writer exits abnormally
+            while self._queue:
+                self._finish(
+                    self._take_batch(), UsageCaptureError("Usage capture stopped before commit acknowledgement")
+                )
+            raise

litellm/proxy/utils.py

@@ -160,10 +160,12 @@ from litellm.litellm_core_utils.core_helpers import (
     independent_snapshot,
     is_expected_client_error,
 )
+from litellm.litellm_core_utils.deployment_pricing import pricing_models
 from litellm.litellm_core_utils.duration_parser import get_budget_window_start
 from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
 from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
 from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
+from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.served_output_texts import (
@@ -278,8 +280,6 @@ from litellm.repositories.user_repository import UserRepository
 from litellm.repositories.verification_token_repository import (
     VerificationTokenRepository,
 )
-from litellm.router_utils.common_utils import resolve_model_group_alias
-from litellm.router_utils.deployment_pricing import pricing_models
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES
 from litellm.types.llms.openai import ResponsesAPIResponse
@@ -3031,14 +3031,21 @@ class ProxyLogging:
         hook to consult, but it judges the payload the same way, so a proxy configured only with
         one of those still has something to say about every record.
         """
-        if request_metadata.get("_guardrail_pipelines"):
+        return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
+
+    def request_has_pre_call_guardrails(
+        self,
+        request_data: Mapping[str, object],
+        event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
+    ) -> bool:
+        if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
             return True
         caps: Final = ProxyLogging.callback_capabilities()
         if caps.has_content_enforcer:
             return True
-        probe: Final = {"metadata": dict(request_metadata)}
         return any(
-            isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
+            isinstance(callback, CustomGuardrail)
+            and callback.should_run_guardrail(data=request_data, event_type=event_type)
             for callback in caps.resolved_callbacks
         )