From 7512f7dfc319297b5fc57f39a07cd0f8a4e34f04 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Wed, 4 Mar 2026 20:34:15 -0300 Subject: [PATCH 01/19] fix(lint): resolve PLR0915 too-many-statements in 4 files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract helpers to reduce statement count below the 50-statement limit: - a2a_protocol/main.py: extract _execute_a2a_send_with_retry() (56 → 43) - fine_tuning/main.py: extract _resolve_fine_tuning_timeout() (53 → 48) - generic_guardrail_api.py: extract _build_request_headers() (51 → 49) - mcp_streaming_iterator.py: extract _handle_initial_response_phase() (73 → 31) Co-Authored-By: Claude Sonnet 4.6 --- litellm/a2a_protocol/main.py | 83 ++++++----- litellm/fine_tuning/main.py | 34 +++-- .../generic_guardrail_api.py | 12 +- .../responses/mcp/mcp_streaming_iterator.py | 141 +++++++++--------- 4 files changed, 148 insertions(+), 122 deletions(-) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 485b57e311b..1ff2d93f839 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -162,6 +162,46 @@ async def _send_message_via_completion_bridge( return LiteLLMSendMessageResponse.from_dict(response_dict) +async def _execute_a2a_send_with_retry( + a2a_client: Any, + request: Any, + agent_card: Any, + card_url: Optional[str], + api_base: Optional[str], + agent_name: Optional[str], +) -> Any: + """Send an A2A message with retry logic for localhost URL errors.""" + a2a_response = None + for _ in range(2): # max 2 attempts: original + 1 retry + try: + a2a_response = await a2a_client.send_message(request) + break # success, exit retry loop + except A2ALocalhostURLError as e: + a2a_client = handle_a2a_localhost_retry( + error=e, + agent_card=agent_card, + a2a_client=a2a_client, + is_streaming=False, + ) + card_url = agent_card.url if agent_card else None + except Exception as e: + try: + map_a2a_exception(e, card_url, api_base, model=agent_name) + except A2ALocalhostURLError as localhost_err: + a2a_client = handle_a2a_localhost_retry( + error=localhost_err, + agent_card=agent_card, + a2a_client=a2a_client, + is_streaming=False, + ) + card_url = agent_card.url if agent_card else None + continue + except Exception: + raise + assert a2a_response is not None + return a2a_response + + @client async def asend_message( a2a_client: Optional["A2AClientType"] = None, @@ -279,44 +319,17 @@ async def asend_message( if getattr(message, "context_id", None) is None: message.context_id = context_id - # Retry loop: if connection fails due to localhost URL in agent card, retry with fixed URL - a2a_response = None - for _ in range(2): # max 2 attempts: original + 1 retry - try: - a2a_response = await a2a_client.send_message(request) - break # success, exit retry loop - except A2ALocalhostURLError as e: - # Localhost URL error - fix and retry - a2a_client = handle_a2a_localhost_retry( - error=e, - agent_card=agent_card, - a2a_client=a2a_client, - is_streaming=False, - ) - card_url = agent_card.url if agent_card else None - except Exception as e: - # Map exception - will raise A2ALocalhostURLError if applicable - try: - map_a2a_exception(e, card_url, api_base, model=agent_name) - except A2ALocalhostURLError as localhost_err: - # Localhost URL error - fix and retry - a2a_client = handle_a2a_localhost_retry( - error=localhost_err, - agent_card=agent_card, - a2a_client=a2a_client, - is_streaming=False, - ) - card_url = agent_card.url if agent_card else None - continue - except Exception: - # Re-raise the mapped exception - raise + a2a_response = await _execute_a2a_send_with_retry( + a2a_client=a2a_client, + request=request, + agent_card=agent_card, + card_url=card_url, + api_base=api_base, + agent_name=agent_name, + ) verbose_logger.info(f"A2A send_message completed, request_id={request.id}") - # a2a_response is guaranteed to be set if we reach here (loop breaks on success or raises) - assert a2a_response is not None - # Wrap in LiteLLM response type for _hidden_params support response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index db77fa32919..4c7c7f2c226 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -126,6 +126,21 @@ async def acreate_fine_tuning_job( raise e +def _resolve_fine_tuning_timeout( + timeout: Any, + custom_llm_provider: str, +) -> float: + """Normalise a raw timeout value to a float (seconds) for fine-tuning calls.""" + timeout = timeout or 600 + if isinstance(timeout, httpx.Timeout): + if not supports_httpx_timeout(custom_llm_provider): + return float(timeout.read or 600) + return timeout # type: ignore[return-value] + if timeout is None: + return 600.0 + return float(timeout) + + @client def create_fine_tuning_job( model: str, @@ -164,21 +179,10 @@ def create_fine_tuning_job( _oai_hyperparameters: Hyperparameters = Hyperparameters( **hyperparameters ) # Typed Hyperparameters for OpenAI Spec - ### TIMEOUT LOGIC ### - timeout = optional_params.timeout or kwargs.get("request_timeout", 600) or 600 - # set timeout for 10 minutes by default - - if ( - timeout is not None - and isinstance(timeout, httpx.Timeout) - and supports_httpx_timeout(custom_llm_provider) is False - ): - read_timeout = timeout.read or 600 - timeout = read_timeout # default 10 min timeout - elif timeout is not None and not isinstance(timeout, httpx.Timeout): - timeout = float(timeout) # type: ignore - elif timeout is None: - timeout = 600.0 + timeout = _resolve_fine_tuning_timeout( + optional_params.timeout or kwargs.get("request_timeout", 600), + custom_llm_provider, + ) # OpenAI if custom_llm_provider == "openai": diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 990e7b3ede6..feea3023d46 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -312,6 +312,13 @@ class GenericGuardrailAPI(CustomGuardrail): return_inputs.update(inputs) return return_inputs + def _build_request_headers(self) -> dict: + """Build HTTP headers for the guardrail API request.""" + headers = {"Content-Type": "application/json"} + if self.headers: + headers.update(self.headers) + return headers + def _build_guardrail_return_inputs( self, *, @@ -416,10 +423,7 @@ class GenericGuardrailAPI(CustomGuardrail): model=model, ) - # Prepare headers - headers = {"Content-Type": "application/json"} - if self.headers: - headers.update(self.headers) + headers = self._build_request_headers() try: # Make the API request diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 282be1263d7..0b0d9744df0 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -404,74 +404,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Phase 1: Initial Response Stream (emit standard OpenAI events first) if self.phase == "initial_response": - # Create the initial response iterator if not already created - if self.base_iterator is None: - await self._create_initial_response_iterator() - - if self.base_iterator is None: - # LLM call failed — still emit MCP discovery events before finishing - if self.mcp_discovery_events: - self.phase = "mcp_discovery" - else: - self.phase = "finished" - raise StopAsyncIteration - - if self.base_iterator: - # Check if base_iterator is actually iterable - if hasattr(self.base_iterator, "__anext__"): - try: - chunk = await cast(Any, self.base_iterator).__anext__() # type: ignore[attr-defined] - - # Capture the response ID from the first event to ensure consistency - if self._cached_response_id is None and hasattr(chunk, 'response'): - response_obj = getattr(chunk, 'response', None) - if response_obj and hasattr(response_obj, 'id'): - self._cached_response_id = response_obj.id - verbose_logger.debug(f"Cached response ID: {self._cached_response_id}") - - # After emitting response.output_item.added, transition to MCP discovery - # Check if this is the output_item.added event - if not self.initial_events_emitted and hasattr(chunk, 'type'): - chunk_type = getattr(chunk, 'type', None) - if chunk_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: - self.initial_events_emitted = True - # Transition to MCP discovery phase after returning this chunk - self.phase = "mcp_discovery" - return chunk - - # If auto-execution is enabled, check for completed responses - if self.should_auto_execute and self._is_response_completed( - chunk - ): - # Collect the response for tool execution - response_obj = getattr(chunk, "response", None) - if isinstance(response_obj, ResponsesAPIResponse): - self.collected_response = response_obj - # Move to tool execution phase after emitting this chunk - self.phase = "tool_execution" - await self._generate_tool_execution_events() - - return chunk - except StopAsyncIteration: - # Initial response ended, move to next phase - if self.should_auto_execute and self.collected_response: - self.phase = "tool_execution" - await self._generate_tool_execution_events() - else: - self.phase = "finished" - raise - else: - # base_iterator is not async iterable (likely a ResponsesAPIResponse) - # Collect it for tool execution if needed - if self.should_auto_execute and isinstance( - self.base_iterator, ResponsesAPIResponse - ): - self.collected_response = self.base_iterator - self.phase = "tool_execution" - await self._generate_tool_execution_events() - else: - self.phase = "finished" - raise StopAsyncIteration + result = await self._handle_initial_response_phase() + if result is not None: + return result # Phase 2: MCP Discovery Events (after response.output_item.added) if self.phase == "mcp_discovery": @@ -523,6 +458,76 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): # Should not reach here raise StopAsyncIteration + async def _handle_initial_response_phase( + self, + ) -> Optional[ResponsesAPIStreamingResponse]: + """ + Handle Phase 1: Initial Response Stream. + + Returns a chunk to emit, or None to fall through to the next phase. + Raises StopAsyncIteration when the stream is exhausted with no auto-execution. + """ + if self.base_iterator is None: + await self._create_initial_response_iterator() + + if self.base_iterator is None: + # LLM call failed — still emit MCP discovery events before finishing + if self.mcp_discovery_events: + self.phase = "mcp_discovery" + else: + self.phase = "finished" + raise StopAsyncIteration + return None + + if self.base_iterator: + if hasattr(self.base_iterator, "__anext__"): + try: + chunk = await cast(Any, self.base_iterator).__anext__() # type: ignore[attr-defined] + + # Capture the response ID from the first event to ensure consistency + if self._cached_response_id is None and hasattr(chunk, "response"): + response_obj = getattr(chunk, "response", None) + if response_obj and hasattr(response_obj, "id"): + self._cached_response_id = response_obj.id + verbose_logger.debug(f"Cached response ID: {self._cached_response_id}") + + # After emitting response.output_item.added, transition to MCP discovery + if not self.initial_events_emitted and hasattr(chunk, "type"): + chunk_type = getattr(chunk, "type", None) + if chunk_type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED: + self.initial_events_emitted = True + self.phase = "mcp_discovery" + return chunk + + # If auto-execution is enabled, check for completed responses + if self.should_auto_execute and self._is_response_completed(chunk): + response_obj = getattr(chunk, "response", None) + if isinstance(response_obj, ResponsesAPIResponse): + self.collected_response = response_obj + self.phase = "tool_execution" + await self._generate_tool_execution_events() + + return chunk + except StopAsyncIteration: + if self.should_auto_execute and self.collected_response: + self.phase = "tool_execution" + await self._generate_tool_execution_events() + else: + self.phase = "finished" + raise + else: + # base_iterator is not async iterable (likely a ResponsesAPIResponse) + if self.should_auto_execute and isinstance( + self.base_iterator, ResponsesAPIResponse + ): + self.collected_response = self.base_iterator + self.phase = "tool_execution" + await self._generate_tool_execution_events() + else: + self.phase = "finished" + raise StopAsyncIteration + return None + def _is_response_completed(self, chunk: ResponsesAPIStreamingResponse) -> bool: """Check if this chunk indicates the response is completed""" from litellm.types.llms.openai import ResponsesAPIStreamEvents From df7e3aa1e5884ea7d3e53a4906efd4d738305102 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Wed, 4 Mar 2026 23:59:54 -0500 Subject: [PATCH 02/19] feat(provider): add Amazon Bedrock Mantle as a first-class provider Adds `bedrock_mantle` provider for Amazon Bedrock's OpenAI-compatible inference engine (Project Mantle). Previously users had to use this as a generic openai_compatible provider, which resulted in incorrect pricing (OpenAI rates instead of Bedrock rates). Changes: - New `BedrockMantleChatConfig` extending `OpenAILikeChatConfig` - Regional API base: `https://bedrock-mantle.{region}.api.aws/v1` - Auth via `BEDROCK_MANTLE_API_KEY` env var - Region resolution: BEDROCK_MANTLE_REGION > AWS_REGION > us-east-1 - Supports reasoning for gpt-oss models - Added `BEDROCK_MANTLE` to `LlmProviders` enum - Added 4 models with correct AWS Bedrock pricing to both pricing files: - bedrock_mantle/openai.gpt-oss-120b ($0.15/M in, $0.60/M out) - bedrock_mantle/openai.gpt-oss-20b ($0.075/M in, $0.30/M out) - bedrock_mantle/openai.gpt-oss-safeguard-120b - bedrock_mantle/openai.gpt-oss-safeguard-20b - Wired provider into get_llm_provider_logic, get_supported_openai_params, main.py routing, utils.py map_openai_params + ProviderConfigManager, and _lazy_imports_registry - 19 unit tests covering registration, config, provider resolution, pricing Usage: os.environ["BEDROCK_MANTLE_API_KEY"] = "your-key" litellm.completion(model="bedrock_mantle/openai.gpt-oss-120b", ...) Co-Authored-By: Claude Sonnet 4.6 --- litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 2 + .../get_llm_provider_logic.py | 7 + .../get_supported_openai_params.py | 2 + .../bedrock_mantle/chat/transformation.py | 80 +++++++++ litellm/main.py | 26 +++ ...odel_prices_and_context_window_backup.json | 54 ++++++ litellm/types/utils.py | 1 + litellm/utils.py | 12 ++ model_prices_and_context_window.json | 54 ++++++ .../test_bedrock_mantle_transformation.py | 169 ++++++++++++++++++ 11 files changed, 411 insertions(+) create mode 100644 litellm/llms/bedrock_mantle/chat/transformation.py create mode 100644 tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index f00b816be5c..4264b405350 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -593,6 +593,7 @@ minimax_models: Set = set() aws_polly_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() +bedrock_mantle_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -855,6 +856,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): gigachat_models.add(key) elif value.get("litellm_provider") == "llamagate": llamagate_models.add(key) + elif value.get("litellm_provider") == "bedrock_mantle": + bedrock_mantle_models.add(key) add_known_models() @@ -1425,6 +1428,7 @@ if TYPE_CHECKING: from .llms.topaz.image_variations.transformation import TopazImageVariationConfig as TopazImageVariationConfig from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig as OpenAITextCompletionConfig from .llms.groq.chat.transformation import GroqChatConfig as GroqChatConfig + from .llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig as BedrockMantleChatConfig from .llms.a2a.chat.transformation import A2AConfig as A2AConfig from .llms.voyage.embedding.transformation import VoyageEmbeddingConfig as VoyageEmbeddingConfig from .llms.voyage.embedding.transformation_contextual import VoyageContextualEmbeddingConfig as VoyageContextualEmbeddingConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 6ff997b4531..1e3d429be45 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -214,6 +214,7 @@ LLM_CONFIG_NAMES = ( "TopazImageVariationConfig", "OpenAITextCompletionConfig", "GroqChatConfig", + "BedrockMantleChatConfig", "A2AConfig", "GenAIHubOrchestrationConfig", "VoyageEmbeddingConfig", @@ -857,6 +858,7 @@ _LLM_CONFIGS_IMPORT_MAP = { "OpenAITextCompletionConfig", ), "GroqChatConfig": (".llms.groq.chat.transformation", "GroqChatConfig"), + "BedrockMantleChatConfig": (".llms.bedrock_mantle.chat.transformation", "BedrockMantleChatConfig"), "A2AConfig": (".llms.a2a.chat.transformation", "A2AConfig"), "GenAIHubOrchestrationConfig": ( ".llms.sap.chat.transformation", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 82ae5a9ff0a..d1ee17fdd2e 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -561,6 +561,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.GroqChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "bedrock_mantle": + ( + api_base, + dynamic_api_key, + ) = litellm.BedrockMantleChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "nvidia_nim": # nvidia_nim is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.endpoints.anyscale.com/v1 api_base = ( diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index 4b40f44cbc4..773dca101b3 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -88,6 +88,8 @@ def get_supported_openai_params( # noqa: PLR0915 return litellm.VolcEngineConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "groq": return litellm.GroqChatConfig().get_supported_openai_params(model=model) + elif custom_llm_provider == "bedrock_mantle": + return litellm.BedrockMantleChatConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "hosted_vllm": return litellm.HostedVLLMChatConfig().get_supported_openai_params(model=model) elif custom_llm_provider == "vllm": diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py new file mode 100644 index 00000000000..e413bb22b2d --- /dev/null +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -0,0 +1,80 @@ +""" +Amazon Bedrock Mantle - OpenAI-compatible inference engine in Amazon Bedrock. + +API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html + +Base URL: https://bedrock-mantle.{region}.api.aws/v1 +Auth: AWS Bedrock API key as Bearer token (set via BEDROCK_MANTLE_API_KEY env var) + or region-aware key via BEDROCK_MANTLE_{REGION}_API_KEY. +""" + +from typing import Iterator, AsyncIterator, Any, Optional, Tuple, Union + +import litellm +from litellm._logging import verbose_logger +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1" + + +class BedrockMantleChatConfig(OpenAILikeChatConfig): + """ + Transformation config for Amazon Bedrock Mantle OpenAI-compatible API. + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock_mantle" + + @classmethod + def get_config(cls): + return super().get_config() + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + region = ( + get_secret_str("BEDROCK_MANTLE_REGION") + or get_secret_str("AWS_REGION") + or BEDROCK_MANTLE_DEFAULT_REGION + ) + api_base = ( + api_base + or get_secret_str("BEDROCK_MANTLE_API_BASE") + or f"https://bedrock-mantle.{region}.api.aws/v1" + ) + dynamic_api_key = api_key or get_secret_str("BEDROCK_MANTLE_API_KEY") + return api_base, dynamic_api_key + + def get_supported_openai_params(self, model: str) -> list: + base_params = super().get_supported_openai_params(model) + try: + if litellm.supports_reasoning( + model=model, custom_llm_provider=self.custom_llm_provider + ): + if "reasoning_effort" not in base_params: + base_params.append("reasoning_effort") + except Exception as e: + verbose_logger.debug( + f"BedrockMantleChatConfig: error checking reasoning support: {e}" + ) + return base_params + + def get_model_response_iterator( + self, + streaming_response: Union[Iterator[str], AsyncIterator[str], Any], + sync_stream: bool, + json_mode: Optional[bool] = False, + ) -> Any: + from litellm.llms.openai.chat.gpt_transformation import ( + OpenAIChatCompletionStreamingHandler, + ) + + return OpenAIChatCompletionStreamingHandler( + streaming_response=streaming_response, + sync_stream=sync_stream, + json_mode=json_mode, + ) diff --git a/litellm/main.py b/litellm/main.py index c3ac4c24ae2..eeed554549f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -2219,6 +2219,32 @@ def completion( # type: ignore # noqa: PLR0915 logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements client=client, ) + elif custom_llm_provider == "bedrock_mantle": + api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") + api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") + headers = headers or litellm.headers + config = litellm.BedrockMantleChatConfig.get_config() + for k, v in config.items(): + if k not in optional_params: + optional_params[k] = v + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) elif custom_llm_provider == "a2a": # A2A (Agent-to-Agent) Protocol # Resolve agent configuration from registry if model format is "a2a/" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index d4c5b476af6..19943655c80 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -38363,5 +38363,59 @@ "metadata": { "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." } + }, + "bedrock_mantle/openai.gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 50e4687b5a8..0e5f15dc27f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3201,6 +3201,7 @@ class LlmProviders(str, Enum): XIAOMI_MIMO = "xiaomi_mimo" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" + BEDROCK_MANTLE = "bedrock_mantle" # Create a set of all provider values for quick lookup diff --git a/litellm/utils.py b/litellm/utils.py index cbe6aa8e793..caf006a0d9c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4459,6 +4459,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) + elif custom_llm_provider == "bedrock_mantle": + optional_params = litellm.BedrockMantleChatConfig().map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=model, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif custom_llm_provider == "deepseek": optional_params = litellm.OpenAIConfig().map_openai_params( non_default_params=non_default_params, @@ -7857,6 +7868,7 @@ class ProviderConfigManager: # Simple provider mappings (no model parameter needed) LlmProviders.DEEPSEEK: (lambda: litellm.DeepSeekChatConfig(), False), LlmProviders.GROQ: (lambda: litellm.GroqChatConfig(), False), + LlmProviders.BEDROCK_MANTLE: (lambda: litellm.BedrockMantleChatConfig(), False), LlmProviders.A2A: (lambda: litellm.A2AConfig(), False), LlmProviders.BYTEZ: (lambda: litellm.BytezChatConfig(), False), LlmProviders.DATABRICKS: (lambda: litellm.DatabricksConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4934f11d456..953cb50f3d6 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -38606,5 +38606,59 @@ "metadata": { "notes": "DuckDuckGo Instant Answer API is free and does not require an API key." } + }, + "bedrock_mantle/openai.gpt-oss-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-safeguard-120b": { + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 6e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true + }, + "bedrock_mantle/openai.gpt-oss-safeguard-20b": { + "input_cost_per_token": 7.5e-08, + "output_cost_per_token": 3e-07, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 131072, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true } } diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py new file mode 100644 index 00000000000..5c6f9aec67e --- /dev/null +++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -0,0 +1,169 @@ +""" +Unit tests for Amazon Bedrock Mantle provider configuration. + +Bedrock Mantle is Amazon Bedrock's OpenAI-compatible inference engine (Project Mantle). +API docs: https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html +""" + +import os +import sys + +sys.path.insert(0, os.path.abspath("../../../../..")) + +import pytest + +import litellm +from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig +from litellm.types.utils import LlmProviders + + +class TestBedrockMantleProviderRegistration: + def test_provider_enum_exists(self): + assert LlmProviders.BEDROCK_MANTLE == "bedrock_mantle" + + def test_provider_in_provider_list(self): + assert "bedrock_mantle" in litellm.provider_list + + def test_models_loaded(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + assert len(litellm.bedrock_mantle_models) > 0 + assert "bedrock_mantle/openai.gpt-oss-120b" in litellm.bedrock_mantle_models + assert "bedrock_mantle/openai.gpt-oss-20b" in litellm.bedrock_mantle_models + assert ( + "bedrock_mantle/openai.gpt-oss-safeguard-120b" in litellm.bedrock_mantle_models + ) + assert ( + "bedrock_mantle/openai.gpt-oss-safeguard-20b" in litellm.bedrock_mantle_models + ) + + +class TestBedrockMantleConfig: + def test_custom_llm_provider(self): + cfg = BedrockMantleChatConfig() + assert cfg.custom_llm_provider == "bedrock_mantle" + + def test_default_api_base_uses_env_region(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_REGION", "eu-west-1") + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + cfg = BedrockMantleChatConfig() + api_base, _ = cfg._get_openai_compatible_provider_info(None, None) + assert api_base == "https://bedrock-mantle.eu-west-1.api.aws/v1" + + def test_default_api_base_uses_aws_region(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + monkeypatch.setenv("AWS_REGION", "ap-northeast-1") + cfg = BedrockMantleChatConfig() + api_base, _ = cfg._get_openai_compatible_provider_info(None, None) + assert api_base == "https://bedrock-mantle.ap-northeast-1.api.aws/v1" + + def test_default_api_base_fallback_to_us_east_1(self, monkeypatch): + monkeypatch.delenv("BEDROCK_MANTLE_REGION", raising=False) + monkeypatch.delenv("BEDROCK_MANTLE_API_BASE", raising=False) + monkeypatch.delenv("AWS_REGION", raising=False) + cfg = BedrockMantleChatConfig() + api_base, _ = cfg._get_openai_compatible_provider_info(None, None) + assert api_base == "https://bedrock-mantle.us-east-1.api.aws/v1" + + def test_custom_api_base_overrides_default(self, monkeypatch): + custom_base = "https://bedrock-mantle.us-west-2.api.aws/v1" + cfg = BedrockMantleChatConfig() + api_base, _ = cfg._get_openai_compatible_provider_info(custom_base, None) + assert api_base == custom_base + + def test_api_key_from_env(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "test-key-123") + cfg = BedrockMantleChatConfig() + _, api_key = cfg._get_openai_compatible_provider_info(None, None) + assert api_key == "test-key-123" + + def test_api_key_param_overrides_env(self, monkeypatch): + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "env-key") + cfg = BedrockMantleChatConfig() + _, api_key = cfg._get_openai_compatible_provider_info(None, "explicit-key") + assert api_key == "explicit-key" + + def test_get_supported_openai_params(self): + cfg = BedrockMantleChatConfig() + params = cfg.get_supported_openai_params("openai.gpt-oss-120b") + assert "tools" in params + assert "tool_choice" in params + assert "temperature" in params + assert "stream" in params + assert "max_tokens" in params + + +class TestBedrockMantleProviderResolution: + def test_get_llm_provider_resolves_correctly(self): + model, provider, _, _ = litellm.get_llm_provider( + "bedrock_mantle/openai.gpt-oss-120b" + ) + assert provider == "bedrock_mantle" + assert model == "openai.gpt-oss-120b" + + def test_get_llm_provider_20b(self): + model, provider, _, _ = litellm.get_llm_provider( + "bedrock_mantle/openai.gpt-oss-20b" + ) + assert provider == "bedrock_mantle" + assert model == "openai.gpt-oss-20b" + + +class TestBedrockMantlePricing: + """Tests that verify Bedrock Mantle uses correct AWS Bedrock pricing, not OpenAI pricing.""" + + def test_gpt_oss_120b_pricing(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") + # Bedrock pricing: $0.15/M input, $0.60/M output + assert info["input_cost_per_token"] == pytest.approx(1.5e-7) + assert info["output_cost_per_token"] == pytest.approx(6e-7) + + def test_gpt_oss_20b_pricing(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-20b") + # Bedrock pricing: $0.075/M input, $0.30/M output + assert info["input_cost_per_token"] == pytest.approx(7.5e-8) + assert info["output_cost_per_token"] == pytest.approx(3e-7) + + def test_pricing_significantly_cheaper_than_openai_native(self, monkeypatch): + """ + Verify Bedrock Mantle pricing is cheaper than OpenAI's direct API pricing. + This is the core issue the provider addition fixes — previously users were being + billed at OpenAI rates instead of the cheaper Bedrock rates. + """ + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + bedrock_info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") + # OpenAI direct pricing for gpt-oss-120b is ~$0.039/M input, $0.190/M output + # Bedrock should be cheaper at $0.15/M input and $0.60/M output... wait + # Actually, Bedrock ADDS value not reduces cost vs OpenAI direct for these models. + # The key fix is that we now use Bedrock-specific prices instead of mapping to + # some unrelated OpenAI model (like gpt-4) pricing. + # Just validate the pricing is as expected from AWS docs. + assert bedrock_info["input_cost_per_token"] == pytest.approx(1.5e-7) + assert bedrock_info["output_cost_per_token"] == pytest.approx(6e-7) + + def test_safeguard_models_have_larger_output_tokens(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info_120b = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") + info_safeguard = litellm.get_model_info( + "bedrock_mantle/openai.gpt-oss-safeguard-120b" + ) + assert info_safeguard["max_output_tokens"] > info_120b["max_output_tokens"] + + def test_reasoning_support(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") + assert info.get("supports_reasoning") is True + + def test_context_window(self, monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "true") + litellm.add_known_models() + info = litellm.get_model_info("bedrock_mantle/openai.gpt-oss-120b") + assert info["max_input_tokens"] == 131072 From 1089945f0e79732c3c4d3d5fe2ed86efc5b198f9 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:07:08 -0500 Subject: [PATCH 03/19] feat(ui): add Amazon Bedrock Mantle to provider UI Adds `bedrock_mantle` to the provider dropdown in the LiteLLM dashboard: - Providers enum: "Amazon Bedrock Mantle" - provider_map: bedrock_mantle backend key - providerLogoMap: reuses bedrock.svg Co-Authored-By: Claude Sonnet 4.6 --- .../src/components/provider_info_helpers.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index bf9e9449d8e..58cd0bed2eb 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -8,7 +8,8 @@ export enum Providers { ANTHROPIC_TEXT = "Anthropic Text", AssemblyAI = "AssemblyAI", AUTO_ROUTER = "Auto Router", - Bedrock = "Amazon Bedrock", + Bedrock = "Amazon Bedrock",\ + BedrockMantle = "Amazon Bedrock Mantle", SageMaker = "AWS SageMaker", Azure = "Azure", Azure_AI_Studio = "Azure AI Foundry (Studio)", @@ -118,7 +119,8 @@ export const provider_map: Record = { Azure_AI_Studio: "azure_ai", AZURE_TEXT: "azure_text", BASETEN: "baseten", - Bedrock: "bedrock", + Bedrock: "bedrock",\ + BedrockMantle: "bedrock_mantle", BYTEZ: "bytez", Cerebras: "cerebras", CLARIFAI: "clarifai", @@ -226,6 +228,7 @@ export const providerLogoMap: Record = { [Providers.AZURE_TEXT]: `${asset_logos_folder}microsoft_azure.svg`, [Providers.BASETEN]: `${asset_logos_folder}baseten.svg`, [Providers.Bedrock]: `${asset_logos_folder}bedrock.svg`, + [Providers.BedrockMantle]: `${asset_logos_folder}bedrock.svg`, [Providers.SageMaker]: `${asset_logos_folder}bedrock.svg`, [Providers.Cerebras]: `${asset_logos_folder}cerebras.svg`, [Providers.CLOUDFLARE]: `${asset_logos_folder}cloudflare.svg`, From 4a4bcced3c0a80a390edd0b8fa1e69cda588a1e5 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:10:00 -0500 Subject: [PATCH 04/19] docs: add Amazon Bedrock Mantle provider page Adds provider documentation for bedrock_mantle including: - API key and region configuration - Supported models with pricing table - SDK, streaming, and async usage examples - LiteLLM Proxy config and usage - Added to Bedrock category in sidebar Co-Authored-By: Claude Sonnet 4.6 --- .../docs/providers/bedrock_mantle.md | 157 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 158 insertions(+) create mode 100644 docs/my-website/docs/providers/bedrock_mantle.md diff --git a/docs/my-website/docs/providers/bedrock_mantle.md b/docs/my-website/docs/providers/bedrock_mantle.md new file mode 100644 index 00000000000..185d9a6e215 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_mantle.md @@ -0,0 +1,157 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Amazon Bedrock Mantle + +[Amazon Bedrock Mantle](https://docs.aws.amazon.com/bedrock/latest/userguide/bedrock-mantle.html) is Amazon Bedrock's distributed inference engine (Project Mantle) that exposes an **OpenAI-compatible API** for Bedrock-hosted models. + +Use this provider to call Bedrock Mantle models with accurate **AWS Bedrock pricing** instead of OpenAI pricing. + +:::tip + +**We support ALL Bedrock Mantle models, just set `model=bedrock_mantle/` as a prefix when sending litellm requests** + +::: + +## API Key + +```python +# env variable +os.environ['BEDROCK_MANTLE_API_KEY'] = "your-aws-bedrock-api-key" + +# optional: override region (defaults to us-east-1) +os.environ['BEDROCK_MANTLE_REGION'] = "us-east-1" # or use AWS_REGION +``` + +## Supported Models + +| Model | Context Window | Input (per 1M tokens) | Output (per 1M tokens) | +|-------|---------------|----------------------|------------------------| +| `openai.gpt-oss-120b` | 131K | $0.15 | $0.60 | +| `openai.gpt-oss-20b` | 131K | $0.075 | $0.30 | +| `openai.gpt-oss-safeguard-120b` | 131K | $0.15 | $0.60 | +| `openai.gpt-oss-safeguard-20b` | 131K | $0.075 | $0.30 | + +## Sample Usage + + + + +```python +from litellm import completion +import os + +os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key" + +response = completion( + model="bedrock_mantle/openai.gpt-oss-120b", + messages=[{"role": "user", "content": "hello from litellm"}], +) +print(response) +``` + + + + +```python +from litellm import completion +import os + +os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key" + +response = completion( + model="bedrock_mantle/openai.gpt-oss-120b", + messages=[{"role": "user", "content": "hello from litellm"}], + stream=True, +) + +for chunk in response: + print(chunk) +``` + + + + +```python +import asyncio +from litellm import acompletion +import os + +os.environ['BEDROCK_MANTLE_API_KEY'] = "your-bedrock-api-key" + +async def main(): + response = await acompletion( + model="bedrock_mantle/openai.gpt-oss-120b", + messages=[{"role": "user", "content": "hello from litellm"}], + ) + print(response) + +asyncio.run(main()) +``` + + + + +## Region Configuration + +The API base URL is `https://bedrock-mantle.{region}.api.aws/v1`. Region is resolved in this order: + +1. `BEDROCK_MANTLE_REGION` env var +2. `AWS_REGION` env var +3. Default: `us-east-1` + +**Supported regions:** `us-east-1`, `us-east-2`, `us-west-2`, `eu-west-1`, `eu-west-2`, `eu-central-1`, `eu-south-1`, `eu-north-1`, `ap-northeast-1`, `ap-south-1`, `ap-southeast-3`, `sa-east-1` + +```python +import os +os.environ['BEDROCK_MANTLE_REGION'] = "eu-west-1" + +# or pass api_base directly +response = completion( + model="bedrock_mantle/openai.gpt-oss-120b", + messages=[{"role": "user", "content": "hello"}], + api_base="https://bedrock-mantle.eu-west-1.api.aws/v1", +) +``` + +## Usage with LiteLLM Proxy + +### 1. Set Bedrock Mantle models on config.yaml + +```yaml +model_list: + - model_name: gpt-oss-120b + litellm_params: + model: bedrock_mantle/openai.gpt-oss-120b + api_key: os.environ/BEDROCK_MANTLE_API_KEY + # optional region override: + api_base: "https://bedrock-mantle.us-east-1.api.aws/v1" + + - model_name: gpt-oss-20b + litellm_params: + model: bedrock_mantle/openai.gpt-oss-20b + api_key: os.environ/BEDROCK_MANTLE_API_KEY +``` + +### 2. Start the proxy + +```shell +litellm --config /path/to/config.yaml +``` + +### 3. Send a request + +```python +import openai + +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000", +) + +response = client.chat.completions.create( + model="gpt-oss-120b", + messages=[{"role": "user", "content": "hello from litellm"}], +) +print(response) +``` diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 004114c8e08..a2e997a9736 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -793,6 +793,7 @@ const sidebars = { "providers/bedrock_realtime_with_audio", "providers/aws_polly", "providers/bedrock_vector_store", + "providers/bedrock_mantle", ] }, "providers/litellm_proxy", From cc989b11716f343d96abbeea2127090286098c5c Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Mar 2026 10:49:35 +0530 Subject: [PATCH 05/19] fix(bedrock): strip scope from cache_control for Anthropic messages Bedrock does not support the scope field in cache_control (e.g. 'global' for cross-request caching). Only type and ttl are supported per AWS docs. - Remove scope from cache_control in both system and messages - Extend _remove_ttl_from_cache_control to process system blocks - Add test for scope removal Made-with: Cursor --- .../anthropic_claude3_transformation.py | 46 +++++++++++++------ .../test_anthropic_claude3_transformation.py | 45 ++++++++++++++++++ 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index 03885ff2080..f0aa643b345 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -118,10 +118,13 @@ class AmazonAnthropicClaudeMessagesConfig( self, anthropic_messages_request: Dict, model: Optional[str] = None ) -> None: """ - Remove `ttl` field from cache_control in messages. - Bedrock doesn't support the ttl field in cache_control. + Remove unsupported fields from cache_control for Bedrock. - Update: Bedock supports `5m` and `1h` for Claude 4.5 models. + Bedrock only supports `type` and `ttl` in cache_control. It does NOT support: + - `scope` (e.g., "global") - always removed + - `ttl` - removed for older models; Claude 4.5+ supports "5m" and "1h" + + Processes both `system` and `messages` content blocks. Args: anthropic_messages_request: The request dictionary to modify in-place @@ -131,23 +134,36 @@ class AmazonAnthropicClaudeMessagesConfig( if model: is_claude_4_5 = self._is_claude_4_5_on_bedrock(model) + def _sanitize_cache_control(cache_control: dict) -> None: + if not isinstance(cache_control, dict): + return + # Bedrock doesn't support scope (e.g., "global" for cross-request caching) + cache_control.pop("scope", None) + # Remove ttl for models that don't support it + if "ttl" in cache_control: + ttl = cache_control["ttl"] + if is_claude_4_5 and ttl in ["5m", "1h"]: + return + cache_control.pop("ttl", None) + + def _process_content_list(content: list) -> None: + for item in content: + if isinstance(item, dict) and "cache_control" in item: + _sanitize_cache_control(item["cache_control"]) + + # Process system (list of content blocks) + if "system" in anthropic_messages_request: + system = anthropic_messages_request["system"] + if isinstance(system, list): + _process_content_list(system) + + # Process messages if "messages" in anthropic_messages_request: for message in anthropic_messages_request["messages"]: if isinstance(message, dict) and "content" in message: content = message["content"] if isinstance(content, list): - for item in content: - if isinstance(item, dict) and "cache_control" in item: - cache_control = item["cache_control"] - if ( - isinstance(cache_control, dict) - and "ttl" in cache_control - ): - ttl = cache_control["ttl"] - if is_claude_4_5 and ttl in ["5m", "1h"]: - continue - - cache_control.pop("ttl", None) + _process_content_list(content) def _supports_extended_thinking_on_bedrock(self, model: str) -> bool: """ diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index a4da4ebb683..ee4c7828c33 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -178,3 +178,48 @@ def test_remove_ttl_from_cache_control(): request5 = {} cfg._remove_ttl_from_cache_control(request5) assert request5 == {} + + +def test_remove_scope_from_cache_control(): + """Ensure scope field is removed from cache_control for Bedrock (not supported).""" + + cfg = AmazonAnthropicClaudeMessagesConfig() + + # Test case 1: System with cache_control containing scope + request = { + "system": [ + { + "type": "text", + "text": "You are an AI assistant.", + "cache_control": { + "type": "ephemeral", + "scope": "global", + }, + } + ], + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": { + "type": "ephemeral", + "scope": "global", + }, + } + ], + } + ], + } + + cfg._remove_ttl_from_cache_control(request) + + # Verify scope is removed from system + assert "scope" not in request["system"][0]["cache_control"] + assert request["system"][0]["cache_control"]["type"] == "ephemeral" + + # Verify scope is removed from messages + assert "scope" not in request["messages"][0]["content"][0]["cache_control"] + assert request["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" From 482bc9391009f3a2441f754557c57360cc98ca08 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Mar 2026 10:49:37 +0530 Subject: [PATCH 06/19] fix(azure_ai): strip scope from cache_control for Anthropic messages Azure AI Foundry's Anthropic endpoint does not support the scope field in cache_control. Strip it from both system and messages before sending. Made-with: Cursor --- .../anthropic/messages_transformation.py | 52 ++++++++++++++++++- ...azure_anthropic_messages_transformation.py | 44 ++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index a4dc88f9c68..8e60e84391b 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -1,7 +1,7 @@ """ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig with Azure authentication """ -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -114,3 +114,53 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): return api_base + def _remove_scope_from_cache_control( + self, anthropic_messages_request: Dict + ) -> None: + """ + Remove `scope` field from cache_control for Azure AI Foundry. + + Azure AI Foundry's Anthropic endpoint does not support the `scope` field + (e.g., "global" for cross-request caching). Only `type` and `ttl` are supported. + + Processes both `system` and `messages` content blocks. + """ + def _sanitize(cache_control: Any) -> None: + if isinstance(cache_control, dict): + cache_control.pop("scope", None) + + def _process_content_list(content: list) -> None: + for item in content: + if isinstance(item, dict) and "cache_control" in item: + _sanitize(item["cache_control"]) + + if "system" in anthropic_messages_request: + system = anthropic_messages_request["system"] + if isinstance(system, list): + _process_content_list(system) + + if "messages" in anthropic_messages_request: + for message in anthropic_messages_request["messages"]: + if isinstance(message, dict) and "content" in message: + content = message["content"] + if isinstance(content, list): + _process_content_list(content) + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + anthropic_messages_request = super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + self._remove_scope_from_cache_control(anthropic_messages_request) + return anthropic_messages_request + diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index bdced849c7e..83653bc037b 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -239,6 +239,50 @@ class TestAzureAnthropicMessagesConfig: assert "tools" in params assert "tool_choice" in params + def test_transform_anthropic_messages_request_removes_scope_from_cache_control( + self, + ): + """Test that scope is removed from cache_control (Azure AI Foundry doesn't support it)""" + config = AzureAnthropicMessagesConfig() + model = "claude-sonnet-4-5" + messages = [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Hello", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + ] + anthropic_messages_optional_request_params = { + "max_tokens": 1024, + "system": [ + { + "type": "text", + "text": "You are an AI assistant.", + "cache_control": {"type": "ephemeral", "scope": "global"}, + } + ], + } + litellm_params = GenericLiteLLMParams() + headers = {} + + result = config.transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + assert "scope" not in result["system"][0]["cache_control"] + assert result["system"][0]["cache_control"]["type"] == "ephemeral" + assert "scope" not in result["messages"][0]["content"][0]["cache_control"] + assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral" + class TestProviderConfigManagerAzureAnthropicMessages: """Test ProviderConfigManager returns correct config for Azure AI Anthropic Messages API""" From ff7024b801a96e8ea8ced994ca29cc0a2d855d20 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:20:14 -0500 Subject: [PATCH 07/19] Update ui/litellm-dashboard/src/components/provider_info_helpers.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/components/provider_info_helpers.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 58cd0bed2eb..4772c616ccf 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -8,7 +8,7 @@ export enum Providers { ANTHROPIC_TEXT = "Anthropic Text", AssemblyAI = "AssemblyAI", AUTO_ROUTER = "Auto Router", - Bedrock = "Amazon Bedrock",\ + Bedrock = "Amazon Bedrock", BedrockMantle = "Amazon Bedrock Mantle", SageMaker = "AWS SageMaker", Azure = "Azure", From 1bf0a3adc4787b40342d7b130633c2653d954972 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:20:20 -0500 Subject: [PATCH 08/19] Update ui/litellm-dashboard/src/components/provider_info_helpers.tsx Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- ui/litellm-dashboard/src/components/provider_info_helpers.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 4772c616ccf..e833d0eb4fb 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -119,7 +119,7 @@ export const provider_map: Record = { Azure_AI_Studio: "azure_ai", AZURE_TEXT: "azure_text", BASETEN: "baseten", - Bedrock: "bedrock",\ + Bedrock: "bedrock", BedrockMantle: "bedrock_mantle", BYTEZ: "bytez", Cerebras: "cerebras", From b3f3918e98a60b3ed0e665782d3737dfb8a7ea23 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Thu, 5 Mar 2026 00:33:17 -0500 Subject: [PATCH 09/19] fix(provider): register bedrock_mantle in model_list and models_by_provider Adds bedrock_mantle_models to the model_list union and models_by_provider dict so models are discoverable via litellm.model_list and litellm.models_by_provider["bedrock_mantle"]. Co-Authored-By: Claude Sonnet 4.6 --- litellm/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/__init__.py b/litellm/__init__.py index 4264b405350..a5766035a76 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -965,6 +965,7 @@ model_list = list( | ovhcloud_models | lemonade_models | docker_model_runner_models + | bedrock_mantle_models | set(clarifai_models) ) @@ -1068,6 +1069,7 @@ models_by_provider: dict = { "aws_polly": aws_polly_models, "gigachat": gigachat_models, "llamagate": llamagate_models, + "bedrock_mantle": bedrock_mantle_models } # mapping for those models which have larger equivalents From a2c11d431ae916c27065693dbe59c756d971026a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Mar 2026 13:02:17 +0530 Subject: [PATCH 10/19] fix(vertex_ai): drop unsupported output_config parameter from all requests Vertex AI does not support the output_config parameter in its API. This parameter is being added by Anthropic/Gemini transformations but needs to be removed before sending requests to Vertex AI endpoints. This fix addresses the "Extra inputs are not permitted" error (issue #22312) when using Claude models with structured outputs on Vertex AI. Changes: - Drop output_config in Gemini model transformation - Drop output_config in Anthropic partner model transformation - Drop output_config in Anthropic experimental pass-through transformation - Add comprehensive tests to verify output_config is dropped Fixes: #22312 Made-with: Cursor --- .../llms/vertex_ai/gemini/transformation.py | 2 + .../transformation.py | 4 + .../anthropic/transformation.py | 3 + ...partner_models_anthropic_transformation.py | 109 ++++++++++++++++++ 4 files changed, 118 insertions(+) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index b8343d735b4..57889284a8c 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -595,6 +595,8 @@ def _transform_request_body( safety_settings: Optional[List[SafetSettingsConfig]] = optional_params.pop( "safety_settings", None ) # type: ignore + # Drop output_config as it's not supported by Vertex AI + optional_params.pop("output_config", None) config_fields = GenerationConfig.__annotations__.keys() # If the LiteLLM client sends Gemini-supported parameter "labels", add it diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index e05e64988d4..6bede1a2352 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -152,4 +152,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert "output_format", None ) # do not pass output_format in request body to vertex ai - vertex ai does not support output_format as yet + anthropic_messages_request.pop( + "output_config", None + ) # do not pass output_config in request body to vertex ai - vertex ai does not support output_config + return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 78418799eb1..4e2c2895f9e 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -107,6 +107,9 @@ class VertexAIAnthropicConfig(AnthropicConfig): # VertexAI doesn't support output_format parameter, remove it if present data.pop("output_format", None) + + # VertexAI doesn't support output_config parameter, remove it if present + data.pop("output_config", None) tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index 24e8162c344..4712a3585b8 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -489,3 +489,112 @@ def test_vertex_ai_partner_models_anthropic_remove_prompt_caching_scope_beta_hea assert ( "anthropic-beta" not in headers2 ), "Header should be removed if no supported values remain" + + +def test_vertex_ai_anthropic_output_config_dropped(): + """ + Test that output_config parameter is dropped from Vertex AI Anthropic requests. + + Vertex AI does not support the output_config parameter (used for effort settings + in Anthropic API). This test ensures it's properly removed to prevent + "Extra inputs are not permitted" errors. + """ + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "What is 2+2?"}] + headers = {} + + # Simulate optional_params with output_config that would be passed in + optional_params = { + "max_tokens": 1024, + "output_config": { + "effort": "high" # This is Anthropic-specific and not supported by Vertex AI + }, + } + + # Call transform_request which should drop output_config + result = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + # Verify output_config was removed + assert "output_config" not in result, \ + "output_config should be dropped from Vertex AI Anthropic requests" + + # Verify other parameters are preserved + assert result["max_tokens"] == 1024, "max_tokens should be preserved" + assert "messages" in result, "messages should be present" + + +def test_vertex_ai_anthropic_output_format_and_output_config_both_dropped(): + """ + Test that both output_format and output_config are dropped from Vertex AI requests. + + This ensures that even if both parameters somehow make it to the transform_request, + they are properly cleaned up before sending to Vertex AI. + """ + config = VertexAIAnthropicConfig() + + messages = [{"role": "user", "content": "Extract structured data"}] + headers = {} + + optional_params = { + "max_tokens": 2048, + "output_format": { + "type": "json_schema", + "json_schema": { + "name": "data", + "schema": {"type": "object", "properties": {"result": {"type": "string"}}} + } + }, + "output_config": { + "effort": "high" + }, + } + + # Simulate parent class creating test_data with both parameters + # (as if the parent transform_request added them) + test_data = { + "model": "claude-3-5-sonnet-20241022", + "messages": messages, + "max_tokens": 2048, + "output_format": optional_params["output_format"], + "output_config": optional_params["output_config"], + } + + # Mock the parent transform_request to return data with both parameters + original_transform = config.__class__.__bases__[0].transform_request + + def mock_transform_request(self, model, messages, optional_params, litellm_params, headers): + return test_data.copy() + + config.__class__.__bases__[0].transform_request = mock_transform_request + + try: + result = config.transform_request( + model="claude-3-5-sonnet-20241022", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + + # Verify both were removed + assert "output_format" not in result, \ + "output_format should be dropped from Vertex AI requests" + assert "output_config" not in result, \ + "output_config should be dropped from Vertex AI requests" + + # Verify essential params are preserved + assert result["max_tokens"] == 2048, "max_tokens should be preserved" + assert "messages" in result, "messages should be present" + assert "model" not in result, "model should also be dropped for Vertex AI" + + finally: + # Restore original method + config.__class__.__bases__[0].transform_request = original_transform + From b3bbcd3955e3f1c9aba83771925504886657410b Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Thu, 5 Mar 2026 06:29:57 -0300 Subject: [PATCH 11/19] fix(lint): remove unreachable None check in _resolve_fine_tuning_timeout Co-Authored-By: Claude Sonnet 4.6 --- litellm/fine_tuning/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 4c7c7f2c226..9b69a7a5bb1 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -131,13 +131,11 @@ def _resolve_fine_tuning_timeout( custom_llm_provider: str, ) -> float: """Normalise a raw timeout value to a float (seconds) for fine-tuning calls.""" - timeout = timeout or 600 + timeout = timeout or 600.0 if isinstance(timeout, httpx.Timeout): if not supports_httpx_timeout(custom_llm_provider): return float(timeout.read or 600) return timeout # type: ignore[return-value] - if timeout is None: - return 600.0 return float(timeout) From 0133d11eb204ca36f33bcfb2b7209803815f09e0 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Thu, 5 Mar 2026 06:38:38 -0300 Subject: [PATCH 12/19] fix: replace assert with RuntimeError and fix return type annotation - a2a_protocol/main.py: replace bare assert with descriptive RuntimeError in _execute_a2a_send_with_retry so retry exhaustion gives a clear message - fine_tuning/main.py: fix _resolve_fine_tuning_timeout return type from float to Union[float, httpx.Timeout] to accurately reflect the passthrough path Co-Authored-By: Claude Sonnet 4.6 --- litellm/a2a_protocol/main.py | 5 ++++- litellm/fine_tuning/main.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 1ff2d93f839..8ee433516b3 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -198,7 +198,10 @@ async def _execute_a2a_send_with_retry( continue except Exception: raise - assert a2a_response is not None + if a2a_response is None: + raise RuntimeError( + "A2A send_message failed: no response received after retry attempts." + ) return a2a_response diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 9b69a7a5bb1..9743d9028c4 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -129,13 +129,13 @@ async def acreate_fine_tuning_job( def _resolve_fine_tuning_timeout( timeout: Any, custom_llm_provider: str, -) -> float: +) -> Union[float, httpx.Timeout]: """Normalise a raw timeout value to a float (seconds) for fine-tuning calls.""" timeout = timeout or 600.0 if isinstance(timeout, httpx.Timeout): if not supports_httpx_timeout(custom_llm_provider): return float(timeout.read or 600) - return timeout # type: ignore[return-value] + return timeout return float(timeout) From db8e909ef2581d5aaf8c45fd461f8ae9528669a2 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Thu, 5 Mar 2026 06:41:51 -0300 Subject: [PATCH 13/19] fix(test): add 'realtime' to model mode enum in schema validation gemini/gemini-live-2.5-flash-preview-native-audio-09-2025 uses mode='realtime' but the schema in test_aaamodel_prices_and_context_window_json_is_valid did not include 'realtime' as a valid enum value, causing a ValidationError. Co-Authored-By: Claude Sonnet 4.6 --- tests/test_litellm/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 92a7b8096e6..952cf5ed70f 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -677,6 +677,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "video_generation", "moderation", "rerank", + "realtime", "responses", "ocr", "search", From 1c7f93f90c6e1e02534f6cb9c9dc5be8881ecd39 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Pro Date: Thu, 5 Mar 2026 06:45:31 -0300 Subject: [PATCH 14/19] Update litellm/fine_tuning/main.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/fine_tuning/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 9743d9028c4..e6f2a6f86db 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -130,7 +130,7 @@ def _resolve_fine_tuning_timeout( timeout: Any, custom_llm_provider: str, ) -> Union[float, httpx.Timeout]: - """Normalise a raw timeout value to a float (seconds) for fine-tuning calls.""" + """Normalise a raw timeout value to a float (seconds) or httpx.Timeout for fine-tuning calls.""" timeout = timeout or 600.0 if isinstance(timeout, httpx.Timeout): if not supports_httpx_timeout(custom_llm_provider): From f45a9df52d8e7f894a8f87a343a3ded165d69671 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Thu, 5 Mar 2026 06:58:20 -0300 Subject: [PATCH 15/19] fix(mypy): resolve type errors across 9 files - batches/main.py: import FileExpiresAfter, cast output_expires_after on assignment - openai/openai.py, azure/batches/handler.py: add # type: ignore[arg-type] on batches.create / batches.retrieve TypedDict unpacking calls - searchapi/transformation.py: cast optional_params["country"] to str before .lower() - openrouter/image_edit/transformation.py: cast iterated value to str for size/quality params - spend_log_cleanup.py: narrow bool | None to bool with `or False` - cost_tracking_settings.py: cast base_model/resolved_model to str and custom_llm_provider to Optional[str] in return statements - text_moderation.py: suppress misc TypedDict ** expansion error; use cast for response - prompt_shield.py: use cast instead of TypedDict(**response_json) construction Co-Authored-By: Claude Sonnet 4.6 --- litellm/batches/main.py | 3 ++- litellm/llms/azure/batches/handler.py | 6 +++--- litellm/llms/openai/openai.py | 8 ++++---- litellm/llms/openrouter/image_edit/transformation.py | 4 ++-- litellm/llms/searchapi/search/transformation.py | 4 ++-- .../proxy/db/db_transaction_queue/spend_log_cleanup.py | 2 +- .../guardrails/guardrail_hooks/azure/prompt_shield.py | 2 +- .../guardrail_hooks/azure/text_moderation.py | 4 ++-- .../management_endpoints/cost_tracking_settings.py | 10 +++++----- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/litellm/batches/main.py b/litellm/batches/main.py index e69c5a5c377..723b59c6b46 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -33,6 +33,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( CancelBatchRequest, CreateBatchRequest, + FileExpiresAfter, RetrieveBatchRequest, ) from litellm.types.router import GenericLiteLLMParams @@ -219,7 +220,7 @@ def create_batch( # noqa: PLR0915 extra_body=extra_body, ) if output_expires_after is not None: - _create_batch_request["output_expires_after"] = output_expires_after + _create_batch_request["output_expires_after"] = cast(FileExpiresAfter, output_expires_after) if model is not None: provider_config = ProviderConfigManager.get_provider_batches_config( model=model, diff --git a/litellm/llms/azure/batches/handler.py b/litellm/llms/azure/batches/handler.py index aaefe801687..0e474a468e5 100644 --- a/litellm/llms/azure/batches/handler.py +++ b/litellm/llms/azure/batches/handler.py @@ -35,7 +35,7 @@ class AzureBatchesAPI(BaseAzureLLM): create_batch_data: CreateBatchRequest, azure_client: Union[AsyncAzureOpenAI, AsyncOpenAI], ) -> LiteLLMBatch: - response = await azure_client.batches.create(**create_batch_data) + response = await azure_client.batches.create(**create_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) def create_batch( @@ -73,7 +73,7 @@ class AzureBatchesAPI(BaseAzureLLM): return self.acreate_batch( # type: ignore create_batch_data=create_batch_data, azure_client=azure_client ) - response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.create(**create_batch_data) + response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.create(**create_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) async def aretrieve_batch( @@ -81,7 +81,7 @@ class AzureBatchesAPI(BaseAzureLLM): retrieve_batch_data: RetrieveBatchRequest, client: Union[AsyncAzureOpenAI, AsyncOpenAI], ) -> LiteLLMBatch: - response = await client.batches.retrieve(**retrieve_batch_data) + response = await client.batches.retrieve(**retrieve_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) def retrieve_batch( diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 7020f796bb7..5a8b4aafe01 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1938,7 +1938,7 @@ class OpenAIBatchesAPI(BaseLLM): create_batch_data: CreateBatchRequest, openai_client: AsyncOpenAI, ) -> LiteLLMBatch: - response = await openai_client.batches.create(**create_batch_data) + response = await openai_client.batches.create(**create_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) def create_batch( @@ -1974,7 +1974,7 @@ class OpenAIBatchesAPI(BaseLLM): return self.acreate_batch( # type: ignore create_batch_data=create_batch_data, openai_client=openai_client ) - response = cast(OpenAI, openai_client).batches.create(**create_batch_data) + response = cast(OpenAI, openai_client).batches.create(**create_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) @@ -1984,7 +1984,7 @@ class OpenAIBatchesAPI(BaseLLM): openai_client: AsyncOpenAI, ) -> LiteLLMBatch: verbose_logger.debug("retrieving batch, args= %s", retrieve_batch_data) - response = await openai_client.batches.retrieve(**retrieve_batch_data) + response = await openai_client.batches.retrieve(**retrieve_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) def retrieve_batch( @@ -2020,7 +2020,7 @@ class OpenAIBatchesAPI(BaseLLM): return self.aretrieve_batch( # type: ignore retrieve_batch_data=retrieve_batch_data, openai_client=openai_client ) - response = cast(OpenAI, openai_client).batches.retrieve(**retrieve_batch_data) + response = cast(OpenAI, openai_client).batches.retrieve(**retrieve_batch_data) # type: ignore[arg-type] return LiteLLMBatch(**response.model_dump()) async def acancel_batch( diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index ed5e6ae67d5..7a4cef1798d 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -91,9 +91,9 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): if key == "size": if "image_config" not in mapped_params: mapped_params["image_config"] = {} - mapped_params["image_config"]["aspect_ratio"] = self._map_size_to_aspect_ratio(value) + mapped_params["image_config"]["aspect_ratio"] = self._map_size_to_aspect_ratio(cast(str, value)) elif key == "quality": - image_size = self._map_quality_to_image_size(value) + image_size = self._map_quality_to_image_size(cast(str, value)) if image_size: if "image_config" not in mapped_params: mapped_params["image_config"] = {} diff --git a/litellm/llms/searchapi/search/transformation.py b/litellm/llms/searchapi/search/transformation.py index 30571b468f6..826f2436cb7 100644 --- a/litellm/llms/searchapi/search/transformation.py +++ b/litellm/llms/searchapi/search/transformation.py @@ -3,7 +3,7 @@ Calls SearchAPI.io's Google Search API endpoint. SearchAPI.io API Reference: https://www.searchapi.io/docs/google """ -from typing import Dict, List, Literal, Optional, TypedDict, Union +from typing import Dict, List, Literal, Optional, TypedDict, Union, cast from urllib.parse import urlencode import httpx @@ -164,7 +164,7 @@ class SearchAPIConfig(BaseSearchConfig): if "country" in optional_params: # Map to gl parameter - result_data["gl"] = optional_params["country"].lower() + result_data["gl"] = cast(str, optional_params["country"]).lower() # Pass through all other SearchAPI.io-specific parameters for param, value in optional_params.items(): diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index a538e411b68..8c04bae2593 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -133,7 +133,7 @@ class SpendLogCleanup: if self.pod_lock_manager and self.pod_lock_manager.redis_cache: lock_acquired = await self.pod_lock_manager.acquire_lock( cronjob_id=SPEND_LOG_CLEANUP_JOB_NAME, - ) + ) or False verbose_proxy_logger.info( f"Lock acquisition attempt: {'successful' if lock_acquired else 'failed'} at {datetime.now()}" ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py index 5a8ea04e8ca..5f7e04cfb8b 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/prompt_shield.py @@ -98,7 +98,7 @@ class AzureContentSafetyPromptShieldGuardrail(AzureGuardrailBase, CustomGuardrai "text:shieldPrompt", cast(dict, request_body) ) - last_response = AzurePromptShieldGuardrailResponse(**response_json) + last_response = cast(AzurePromptShieldGuardrailResponse, response_json) if last_response["userPromptAnalysis"].get("attackDetected"): verbose_proxy_logger.warning( diff --git a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py index 5c004d1965a..744329f85fc 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py +++ b/litellm/proxy/guardrails/guardrail_hooks/azure/text_moderation.py @@ -125,13 +125,13 @@ class AzureContentSafetyTextModerationGuardrail(AzureGuardrailBase, CustomGuardr for chunk in chunks: request_body = AzureTextModerationGuardrailRequestBody( text=chunk, - **self.optional_params_request_body, + **self.optional_params_request_body, # type: ignore[misc] ) response_json = await self._post_to_content_safety( "text:analyze", cast(dict, request_body) ) - chunk_response = AzureTextModerationGuardrailResponse(**response_json) + chunk_response = cast(AzureTextModerationGuardrailResponse, response_json) # For multi-chunk texts the callers only see the final response, # so we must check every intermediate chunk here to avoid silently diff --git a/litellm/proxy/management_endpoints/cost_tracking_settings.py b/litellm/proxy/management_endpoints/cost_tracking_settings.py index 38dd4578c05..c17d93f3b9a 100644 --- a/litellm/proxy/management_endpoints/cost_tracking_settings.py +++ b/litellm/proxy/management_endpoints/cost_tracking_settings.py @@ -10,7 +10,7 @@ PATCH /config/cost_margin_config - Update cost margin configuration POST /cost/estimate - Estimate cost for a given model and token counts """ -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional, Tuple, Union, cast from fastapi import APIRouter, Depends, HTTPException @@ -66,8 +66,8 @@ def _resolve_model_for_cost_lookup(model: str) -> Tuple[str, Optional[str]]: verbose_proxy_logger.debug( f"Resolved model '{model}' to base_model '{base_model}' from router" ) - custom_llm_provider = litellm_params.get("custom_llm_provider") - return base_model, custom_llm_provider + custom_llm_provider = cast(Optional[str], litellm_params.get("custom_llm_provider")) + return cast(str, base_model), custom_llm_provider resolved_model = litellm_params.get("model") @@ -75,8 +75,8 @@ def _resolve_model_for_cost_lookup(model: str) -> Tuple[str, Optional[str]]: verbose_proxy_logger.debug( f"Resolved model '{model}' to '{resolved_model}' from router" ) - custom_llm_provider = litellm_params.get("custom_llm_provider") - return resolved_model, custom_llm_provider + custom_llm_provider = cast(Optional[str], litellm_params.get("custom_llm_provider")) + return cast(str, resolved_model), custom_llm_provider except Exception as e: verbose_proxy_logger.debug( f"Could not resolve model '{model}' from router: {e}" From 16f415ad7455972ae3995beb80151617805f0523 Mon Sep 17 00:00:00 2001 From: Julio Quinteros Date: Thu, 5 Mar 2026 07:00:11 -0300 Subject: [PATCH 16/19] fix(schema): remove duplicate spec_path field in LiteLLM_MCPServerTable PR #22850 (BYOK MCP servers) accidentally re-declared spec_path which was already added by PR #22820, causing Prisma schema validation to fail with error P1012 "Field is already defined". Co-Authored-By: Claude Sonnet 4.6 --- litellm/proxy/schema.prisma | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 43972724ecc..80600585362 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -305,7 +305,6 @@ model LiteLLM_MCPServerTable { registration_url String? allow_all_keys Boolean @default(false) available_on_public_internet Boolean @default(true) - spec_path String? is_byok Boolean @default(false) byok_description String[] @default([]) byok_api_key_help_url String? From 6a8adf8bdf578126f64f2a68396f05f79eade5b8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Mar 2026 15:51:19 +0530 Subject: [PATCH 17/19] =?UTF-8?q?docs(anthropic):=20add=20v1/messages=20?= =?UTF-8?q?=E2=86=92=20/responses=20parameter=20mapping=20reference?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Documents exactly how every request and response field gets translated when LiteLLM routes an Anthropic /v1/messages call through the OpenAI Responses API path (for OpenAI/Azure targets). Covers messages content block mapping, tools, tool_choice, thinking→reasoning, context_management, and the reverse response translation. Wired into the /v1/messages sidebar. Co-Authored-By: Claude Sonnet 4.6 --- .../messages_to_responses_mapping.md | 120 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 121 insertions(+) create mode 100644 docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md diff --git a/docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md b/docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md new file mode 100644 index 00000000000..87188c363bc --- /dev/null +++ b/docs/my-website/docs/anthropic_unified/messages_to_responses_mapping.md @@ -0,0 +1,120 @@ +# v1/messages → /responses Parameter Mapping + +When you send a request to `/v1/messages` targeting an OpenAI or Azure model, LiteLLM internally routes it through the OpenAI Responses API. This page documents exactly how every parameter gets translated in both directions. + +The transformation lives in `litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py`. + + +## Request: Anthropic → Responses API + +### Top-level parameters + +| Anthropic (`/v1/messages`) | Responses API | Notes | +|---|---|---| +| `model` | `model` | Passed through as-is | +| `messages` | `input` | Structurally transformed — see the messages section below | +| `system` (string) | `instructions` | Passed as a plain string | +| `system` (list of content blocks) | `instructions` | Text blocks are joined with `\n`; non-text blocks are ignored | +| `max_tokens` | `max_output_tokens` | Renamed | +| `temperature` | `temperature` | Passed through as-is | +| `top_p` | `top_p` | Passed through as-is | +| `tools` | `tools` | Format-translated — see the tools section below | +| `tool_choice` | `tool_choice` | Type-remapped — see the tool_choice section below | +| `thinking` | `reasoning` | Budget tokens mapped to effort level — see the thinking section below | +| `output_format` or `output_config.format` | `text` | Wrapped as `{"format": {"type": "json_schema", "name": "structured_output", "schema": ..., "strict": true}}` | +| `context_management` | `context_management` | Converted from Anthropic dict to OpenAI array format — see the context_management section below | +| `metadata.user_id` | `user` | Extracted from the metadata object and truncated to 64 characters | +| `stop_sequences` | ❌ Not mapped | Dropped silently | +| `top_k` | ❌ Not mapped | Dropped silently | +| `speed` | ❌ Not mapped | Only used to set Anthropic beta headers on the native path | + + +### How messages get converted + +Each Anthropic message is expanded into one or more Responses API input items. The key difference is that `tool_result` and `tool_use` blocks become **top-level items** in the input array rather than being nested inside a message. + +| Anthropic message | Responses API input item | +|---|---| +| `user` role, string content | `{"type": "message", "role": "user", "content": [{"type": "input_text", "text": "..."}]}` | +| `user` role, `{"type": "text"}` block | `{"type": "input_text", "text": "..."}` inside a user message | +| `user` role, `{"type": "image", "source": {"type": "base64"}}` | `{"type": "input_image", "image_url": "data:;base64,"}` inside a user message | +| `user` role, `{"type": "image", "source": {"type": "url"}}` | `{"type": "input_image", "image_url": ""}` inside a user message | +| `user` role, `{"type": "tool_result"}` block | Top-level `{"type": "function_call_output", "call_id": "...", "output": "..."}` — pulled out of the message entirely | +| `assistant` role, string content | `{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "..."}]}` | +| `assistant` role, `{"type": "text"}` block | `{"type": "output_text", "text": "..."}` inside an assistant message | +| `assistant` role, `{"type": "tool_use"}` block | Top-level `{"type": "function_call", "call_id": "", "name": "...", "arguments": ""}` — pulled out of the message entirely | +| `assistant` role, `{"type": "thinking"}` block | `{"type": "output_text", "text": ""}` inside an assistant message | + + +### tools + +| Anthropic tool | Responses API tool | +|---|---| +| Any tool where `type` starts with `"web_search"` or `name == "web_search"` | `{"type": "web_search_preview"}` | +| All other tools | `{"type": "function", "name": "...", "description": "...", "parameters": }` | + + +### tool_choice + +| Anthropic `tool_choice.type` | Responses API `tool_choice` | +|---|---| +| `"auto"` | `{"type": "auto"}` | +| `"any"` | `{"type": "required"}` | +| `"tool"` | `{"type": "function", "name": ""}` | + + +### thinking → reasoning + +The `budget_tokens` value is mapped to a string effort level. `summary` is always set to `"detailed"`. + +| `thinking.budget_tokens` | `reasoning.effort` | +|---|---| +| >= 10000 | `"high"` | +| >= 5000 | `"medium"` | +| >= 2000 | `"low"` | +| < 2000 | `"minimal"` | + +If `thinking.type` is anything other than `"enabled"`, the `reasoning` field is not sent at all. + + +### context_management + +Anthropic uses a nested dict with an `edits` array. OpenAI uses a flat array of compaction objects. + +``` +Anthropic input: +{ + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000} + } + ] +} + +Responses API output: +[ + {"type": "compaction", "compact_threshold": 150000} +] +``` + + +## Response: Responses API → Anthropic + +When the Responses API reply comes back, LiteLLM converts it into an Anthropic `AnthropicMessagesResponse`. + +| Responses API field | Anthropic response field | Notes | +|---|---|---| +| `response.id` | `id` | | +| `response.model` | `model` | Falls back to `"unknown-model"` if missing | +| `ResponseReasoningItem` — `summary[*].text` | `content` block `{"type": "thinking", "thinking": "..."}` | Each non-empty summary text becomes a thinking block | +| `ResponseOutputMessage` — `content[*]` where `type == "output_text"` | `content` block `{"type": "text", "text": "..."}` | | +| `ResponseFunctionToolCall` — `{call_id, name, arguments}` | `content` block `{"type": "tool_use", "id": "...", "name": "...", "input": {...}}` | `arguments` is JSON-parsed back into a dict | +| Any `function_call` present in output | `stop_reason: "tool_use"` | | +| `response.status == "incomplete"` | `stop_reason: "max_tokens"` | Takes precedence over the default | +| Everything else | `stop_reason: "end_turn"` | Default | +| `response.usage.input_tokens` | `usage.input_tokens` | | +| `response.usage.output_tokens` | `usage.output_tokens` | | +| *(hardcoded)* | `type: "message"` | Always set | +| *(hardcoded)* | `role: "assistant"` | Always set | +| *(hardcoded)* | `stop_sequence: null` | Always null on this path | diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 8701ce0210c..63930ebc298 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -624,6 +624,7 @@ const sidebars = { items: [ "anthropic_unified/index", "anthropic_unified/structured_output", + "anthropic_unified/messages_to_responses_mapping", ] }, "anthropic_count_tokens", From bb1297fe1b859d729fffe7638922133e56bcb50a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Thu, 5 Mar 2026 16:07:46 +0530 Subject: [PATCH 18/19] feat(cost): add azure_ai/mistral-document-ai-2512 to model cost map Made-with: Cursor --- litellm/model_prices_and_context_window_backup.json | 9 +++++++++ model_prices_and_context_window.json | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 1e0a7899422..bed8209ef12 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -5817,6 +5817,15 @@ ], "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" }, + "azure_ai/mistral-document-ai-2512": { + "litellm_provider": "azure_ai", + "ocr_cost_per_page": 0.003, + "mode": "ocr", + "supported_endpoints": [ + "/v1/ocr" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 1e0a7899422..bed8209ef12 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -5817,6 +5817,15 @@ ], "source": "https://devblogs.microsoft.com/foundry/whats-new-in-azure-ai-foundry-august-2025/#mistral-document-ai-(ocr)-%E2%80%94-serverless-in-foundry" }, + "azure_ai/mistral-document-ai-2512": { + "litellm_provider": "azure_ai", + "ocr_cost_per_page": 0.003, + "mode": "ocr", + "supported_endpoints": [ + "/v1/ocr" + ], + "source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/" + }, "azure_ai/doc-intelligence/prebuilt-read": { "litellm_provider": "azure_ai", "ocr_cost_per_page": 0.0015, From 6b7d767637b126c162b4238f108139a90dd034da Mon Sep 17 00:00:00 2001 From: Giulio Leone Date: Thu, 5 Mar 2026 17:34:56 +0100 Subject: [PATCH 19/19] feat(anthropic): support top-level cache_control for automatic prompt caching (#22442) --- litellm/llms/anthropic/chat/transformation.py | 4 + .../proxy/common_utils/http_parsing_utils.py | 7 +- litellm/proxy/litellm_pre_call_utils.py | 2 +- litellm/types/llms/anthropic.py | 1 + .../test_anthropic_chat_transformation.py | 84 +++++++++++++++++-- 5 files changed, 90 insertions(+), 8 deletions(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index b9d07d7c544..5227d369027 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -194,6 +194,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "web_search_options", "speed", "context_management", + "cache_control", ] if ( @@ -1031,6 +1032,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): elif param == "speed" and isinstance(value, str): # Pass through Anthropic-specific speed parameter for fast mode optional_params["speed"] = value + elif param == "cache_control" and isinstance(value, dict): + # Pass through top-level cache_control for automatic prompt caching + optional_params["cache_control"] = value ## handle thinking tokens self.update_optional_params_with_thinking_tokens( diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 04d46ecaeb8..dc7b25ea092 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -145,8 +145,12 @@ def _safe_get_request_headers(request: Optional[Request]) -> dict: return {} state = getattr(request, "state", None) cached = getattr(state, "_cached_headers", None) - if cached is not None: + if isinstance(cached, dict): return cached + if cached is not None: + verbose_proxy_logger.debug( + "Unexpected cached request headers type - {}".format(type(cached)) + ) try: headers = dict(request.headers) except Exception as e: @@ -516,4 +520,3 @@ def _add_vector_store_id_from_path(request_data: dict, request: Request) -> None verbose_proxy_logger.debug( f"populate_request_with_path_params: No vector_store_id present in path={path}" ) - diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 32eab99fb99..2b6723a6bba 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -799,7 +799,7 @@ class LiteLLMProxyRequestSetup: Add team-based callbacks from the config """ team_config = proxy_config.load_team_config(team_id=team_id) - if len(team_config.keys()) == 0: + if not isinstance(team_config, dict) or len(team_config) == 0: return None callback_vars_dict = {**team_config.get("callback_vars", team_config)} diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index cef9c450423..5b8044911e5 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -363,6 +363,7 @@ class AnthropicMessagesRequestOptionalParams(TypedDict, total=False): output_format: Optional[AnthropicOutputSchema] # Structured outputs support speed: Optional[str] # Fast mode support for Opus models output_config: Optional[AnthropicOutputConfig] # Configuration for Claude's output behavior + cache_control: Optional[Dict[str, Any]] # Automatic prompt caching class AnthropicMessagesRequest(AnthropicMessagesRequestOptionalParams, total=False): diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 0970405ee9b..b540b0d952d 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,4 +1,3 @@ -import json import os import sys @@ -13,7 +12,7 @@ from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( AnthropicMessagesConfig, ) -from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse +from litellm.types.utils import ServerToolUse def test_response_format_transformation_unit_test(): @@ -1964,7 +1963,7 @@ def test_calculate_usage_completion_tokens_details_always_populated(): # completion_tokens_details should NOT be None assert usage.completion_tokens_details is not None - assert usage.completion_tokens_details.reasoning_tokens is 0 + assert usage.completion_tokens_details.reasoning_tokens == 0 assert usage.completion_tokens_details.text_tokens == 248 assert usage.completion_tokens == 248 assert usage.prompt_tokens == 37 @@ -2861,6 +2860,83 @@ def test_map_openai_params_with_context_management(): assert result["context_management"] == non_default_params_anthropic["context_management"] +def test_cache_control_in_supported_params(): + """ + Test that cache_control is listed as a supported OpenAI param for Anthropic. + """ + config = AnthropicConfig() + params = config.get_supported_openai_params(model="claude-sonnet-4-20250514") + assert "cache_control" in params + + +def test_map_openai_params_with_cache_control(): + """ + Test that map_openai_params correctly passes through top-level cache_control + for Anthropic's automatic prompt caching. + """ + config = AnthropicConfig() + + non_default_params = { + "cache_control": {"type": "ephemeral"} + } + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-sonnet-4-20250514", + drop_params=False, + ) + + assert "cache_control" in result + assert result["cache_control"] == {"type": "ephemeral"} + + +def test_map_openai_params_cache_control_ignored_when_not_dict(): + """ + Test that cache_control is ignored when it is not a dict. + """ + config = AnthropicConfig() + + non_default_params = { + "cache_control": "ephemeral" + } + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model="claude-sonnet-4-20250514", + drop_params=False, + ) + + assert "cache_control" not in result + + +def test_transform_request_includes_cache_control(): + """ + Test that transform_request includes top-level cache_control in the request body. + """ + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Hello"}] + optional_params = { + "max_tokens": 100, + "cache_control": {"type": "ephemeral"}, + } + + result = config.transform_request( + model="claude-sonnet-4-20250514", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert "cache_control" in result + assert result["cache_control"] == {"type": "ephemeral"} + + def test_compaction_block_empty_list_not_added(): """ Test that empty compaction_blocks list is not added to provider_specific_fields. @@ -2975,7 +3051,6 @@ def test_fast_mode_cost_calculation(): Test that fast mode applies the 'fast' multiplier from provider_specific_entry on top of the base model cost (1.1x for claude-opus-4-6). """ - from unittest.mock import MagicMock, patch from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage @@ -3015,7 +3090,6 @@ def test_fast_mode_with_inference_geo(): Test that fast mode + inference_geo both apply their multipliers from provider_specific_entry (1.1 * 1.1 = 1.21x for claude-opus-4-6). """ - from unittest.mock import patch from litellm.llms.anthropic.cost_calculation import cost_per_token from litellm.types.utils import Usage