diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 8d23526b41a..98322aecf6a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -41,7 +41,11 @@ from litellm.responses.sse_output_recovery import ( record_output_item_chunk, record_output_text_chunk, ) -from litellm.responses.utils import ResponsesAPIRequestUtils, normalize_responses_api_stream_options +from litellm.responses.utils import ( + ResponsesAPIRequestUtils, + auto_reasoning_summary, + normalize_responses_api_stream_options, +) from litellm.types.llms.openai import ( ChatCompletionAnnotation, ChatCompletionReasoningItem, @@ -176,6 +180,16 @@ def _without_audio_input_parts(input_items: list[object]) -> list[object]: return [_without_audio_input_parts_in_item(item) for item in input_items] +def _effort_with_requested_summary(nested_effort: object, top_level_effort: object) -> object: + match top_level_effort, nested_effort: + case {"summary": str() as summary}, str() as effort: + return {"effort": effort, "summary": summary} + case {"summary": str() as summary}, Mapping() as reasoning if "summary" not in reasoning: + return {**reasoning, "summary": summary} + case _: + return nested_effort + + def _served_provider(litellm_params: Mapping[str, object]) -> str | None: custom_llm_provider: Final = litellm_params.get("custom_llm_provider") return custom_llm_provider if isinstance(custom_llm_provider, str) else None @@ -953,6 +967,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): optional_params: dict, responses_api_request: "ResponsesAPIOptionalRequestParams", drop_params: bool = False, + model: str | None = None, + custom_llm_provider: str | None = None, ) -> None: """Map optional_params into responses_api_request (mutates in place).""" for key, value in optional_params.items(): @@ -984,7 +1000,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): elif key == "previous_response_id": responses_api_request["previous_response_id"] = value elif key == "reasoning_effort": - responses_api_request["reasoning"] = self.map_reasoning_effort(value) + responses_api_request["reasoning"] = self.map_reasoning_effort(value, model, custom_llm_provider) elif key == "web_search_options": self._add_web_search_tool(responses_api_request, value) @@ -1100,6 +1116,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): optional_params, responses_api_request, drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params), + model=model, + custom_llm_provider=_served_provider(litellm_params), ) stream: Final = optional_params.get("stream") or litellm_params.get("stream", False) @@ -1705,14 +1723,22 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # Extract supported params from extra_body and merge into optional_params extra_body_copy: Final = extra_body.copy() + top_level_effort: Final = cast(object, optional_params.get("reasoning_effort")) # cast-ok: upcast for key, value in extra_body_copy.items(): if key in supported_responses_api_params: # Prefer extra_body value if it exists (may have more complete info like summary in reasoning_effort) - optional_params[key] = extra_body.pop(key) + extra_body.pop(key) + optional_params[key] = ( + _effort_with_requested_summary(cast(object, value), top_level_effort) # cast-ok: upcast + if key == "reasoning_effort" + else value + ) return optional_params - def map_reasoning_effort(self, reasoning_effort: object) -> Reasoning: + def map_reasoning_effort( + self, reasoning_effort: object, model: str | None = None, custom_llm_provider: str | None = None + ) -> Reasoning: # If dict is passed, convert it directly to Reasoning object if isinstance(reasoning_effort, dict): return Reasoning( @@ -1726,7 +1752,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) return ( - Reasoning(effort=reasoning_effort, summary="detailed") + Reasoning( + effort=reasoning_effort, + summary="detailed" if model is None else auto_reasoning_summary(model, custom_llm_provider), + ) if auto_summary_enabled else Reasoning(effort=reasoning_effort) ) diff --git a/litellm/llms/anthropic/pass_through/adapters/handler.py b/litellm/llms/anthropic/pass_through/adapters/handler.py index 7b8c2550838..a9aa3de69ad 100644 --- a/litellm/llms/anthropic/pass_through/adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/adapters/handler.py @@ -501,7 +501,11 @@ class LiteLLMMessagesToCompletionTransformationHandler: return None if resolved_provider == "litellm_proxy": return None - from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check + from litellm.responses.bridge_check import ( + chat_target_accepts_reasoning_summary, + responses_api_bridge_check, + str_keyed_mapping, + ) if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider): return _ChatTargetReasoning(effort=effort, summary=requested_summary) @@ -510,6 +514,7 @@ class LiteLLMMessagesToCompletionTransformationHandler: web_search_options: Final = completion_kwargs.get("web_search_options") tools: Final = completion_kwargs.get("tools") + client_summary: Final = thinking.get("summary") model_info, _ = responses_api_bridge_check( model=local_model, custom_llm_provider=resolved_provider, @@ -517,8 +522,9 @@ class LiteLLMMessagesToCompletionTransformationHandler: cast(OpenAIWebSearchOptions, web_search_options) if isinstance(web_search_options, dict) else None ), tools=cast("list[dict[str, object]]", tools) if isinstance(tools, list) else None, - reasoning_effort=reasoning_effort, + reasoning_effort=effort if client_summary is None else {"effort": effort, "summary": client_summary}, api_base=resolved_api_base, + deployment_model_info=str_keyed_mapping(completion_kwargs.get("model_info")), ) return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort) diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index 3aad0be1913..0d3e48c49fd 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -14,6 +14,7 @@ from litellm.llms.anthropic.pass_through.utils import ( json_copy_or_original, prompt_cache_key_from_user_id, ) +from litellm.responses.utils import auto_reasoning_summary # OpenAI has a 64-character limit for function/tool names # Anthropic does not have this limit, so we need to truncate long names @@ -736,7 +737,7 @@ class LiteLLMAnthropicMessagesAdapter: if reasoning_effort: return { "reasoning_effort": LiteLLMAnthropicMessagesAdapter._apply_reasoning_summary_wrapping( - reasoning_effort, thinking + reasoning_effort, thinking, model, None ) } return {} @@ -745,6 +746,8 @@ class LiteLLMAnthropicMessagesAdapter: def _apply_reasoning_summary_wrapping( reasoning_effort: str, thinking: Mapping[str, object], + model: str, + custom_llm_provider: str | None, ) -> Any: """ Apply the reasoning_effort/summary wrapping rules shared by every @@ -762,7 +765,7 @@ class LiteLLMAnthropicMessagesAdapter: if summary: return {"effort": reasoning_effort, "summary": summary} if is_reasoning_auto_summary_enabled(): - return {"effort": reasoning_effort, "summary": "detailed"} + return {"effort": reasoning_effort, "summary": auto_reasoning_summary(model, custom_llm_provider)} return reasoning_effort def translate_anthropic_tool_choice_to_openai( @@ -1174,7 +1177,9 @@ class LiteLLMAnthropicMessagesAdapter: new_kwargs["reasoning_effort"] = ( reasoning_effort if is_claude_target - else self._apply_reasoning_summary_wrapping(reasoning_effort, cast(dict[str, object], thinking)) + else self._apply_reasoning_summary_wrapping( + reasoning_effort, cast(dict[str, object], thinking), model, custom_llm_provider + ) ) def _translate_output_format_to_openai( diff --git a/litellm/llms/anthropic/pass_through/messages/handler.py b/litellm/llms/anthropic/pass_through/messages/handler.py index 00bf1774dd0..88cfeaad66a 100644 --- a/litellm/llms/anthropic/pass_through/messages/handler.py +++ b/litellm/llms/anthropic/pass_through/messages/handler.py @@ -55,15 +55,19 @@ _SAFEGUARD_KWARGS: Final = TypeAdapter(Mapping[str, object]) _SAFEGUARD_MESSAGES: Final = TypeAdapter(Sequence[Mapping[str, object]]) -def _bridges_to_responses_api(model: str, custom_llm_provider: str) -> bool: - from litellm.responses.bridge_check import responses_api_bridge_check +def _bridges_to_responses_api(model: str, custom_llm_provider: str, deployment_model_info: object) -> bool: + from litellm.responses.bridge_check import responses_api_bridge_check, str_keyed_mapping - model_info, _ = responses_api_bridge_check(model=model, custom_llm_provider=custom_llm_provider) + model_info, _ = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + deployment_model_info=str_keyed_mapping(deployment_model_info), + ) return model_info.get("mode") == "responses" def _responses_mode_is_lost_by_prefix_strip( - requested_model: str, resolved_model: str, custom_llm_provider: str + requested_model: str, resolved_model: str, custom_llm_provider: str, deployment_model_info: object ) -> bool: """Whether a Responses-only deployment stops looking like one once its provider prefix is stripped. @@ -73,15 +77,16 @@ def _responses_mode_is_lost_by_prefix_strip( """ if requested_model == resolved_model: return False - return _bridges_to_responses_api(requested_model, custom_llm_provider) and not _bridges_to_responses_api( - resolved_model, custom_llm_provider - ) + return _bridges_to_responses_api( + requested_model, custom_llm_provider, deployment_model_info + ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info) def _should_route_to_responses_api( custom_llm_provider: str | None, requested_model: str | None = None, resolved_model: str | None = None, + deployment_model_info: object = None, ) -> bool: """Return True when the request should use the Responses API path. @@ -94,7 +99,9 @@ def _should_route_to_responses_api( return True if custom_llm_provider is None or requested_model is None or resolved_model is None: return False - return _responses_mode_is_lost_by_prefix_strip(requested_model, resolved_model, custom_llm_provider) + return _responses_mode_is_lost_by_prefix_strip( + requested_model, resolved_model, custom_llm_provider, deployment_model_info + ) async def _with_native_safeguard_results( @@ -606,7 +613,7 @@ def anthropic_messages_handler( if anthropic_messages_provider_config is None: # Route to Responses API for OpenAI / Azure, chat/completions for everything else. if kwargs.get("compaction") is None and _should_route_to_responses_api( - custom_llm_provider, original_model, model + custom_llm_provider, original_model, model, kwargs.get("model_info") ): return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( max_tokens=max_tokens, diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py index 692e5d4c175..05ea4e4301d 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/handler.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/handler.py @@ -16,6 +16,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( ) from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint +from litellm.responses.utils import auto_reasoning_summary from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS from litellm.types.llms.anthropic import ( AllAnthropicMessageValues, @@ -188,6 +189,7 @@ def _build_responses_kwargs( include_encrypted_reasoning=_provider_returns_encrypted_reasoning( model, forwarded_kwargs.get("custom_llm_provider") ), + auto_summary_level=auto_reasoning_summary(model, forwarded_kwargs.get("custom_llm_provider")), ) # Normalize reasoning effort based on model capabilities diff --git a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py index 97f2db36a85..fb611b64dda 100644 --- a/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/responses_adapters/transformation.py @@ -544,6 +544,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: def translate_thinking_to_reasoning( thinking: dict[str, object], output_config: dict[str, object] | None = None, + auto_summary_level: str = "detailed", ) -> dict[str, object] | None: """ Convert Anthropic thinking param to Responses API reasoning param. @@ -576,7 +577,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if summary: result["summary"] = summary elif auto_summary: - result["summary"] = "detailed" + result["summary"] = auto_summary_level return result @classmethod @@ -591,6 +592,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: self, anthropic_request: AnthropicMessagesRequest, include_encrypted_reasoning: bool = True, + auto_summary_level: str = "detailed", ) -> dict[str, Any]: """ Translate a full Anthropic /v1/messages request dict to @@ -599,7 +601,8 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ``include_encrypted_reasoning`` asks the provider for ``reasoning.encrypted_content`` on every call, so a reasoning model's items can be replayed intact next turn even when the client sent no ``thinking`` block; pass False for a provider whose - Responses API rejects ``include``. + Responses API rejects ``include``. ``auto_summary_level`` is the ``reasoning.summary`` the + opt-in auto summary injects when the client sent none. """ model: Final[str] = anthropic_request["model"] messages_list: Final = cast( @@ -670,6 +673,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: reasoning: Final = self.translate_thinking_to_reasoning( thinking, output_config=cast(dict[str, Any] | None, output_config), + auto_summary_level=auto_summary_level, ) if reasoning: responses_kwargs["reasoning"] = reasoning diff --git a/litellm/llms/base_llm/responses/transformation.py b/litellm/llms/base_llm/responses/transformation.py index ed8bfc8cb96..01baeed9bef 100644 --- a/litellm/llms/base_llm/responses/transformation.py +++ b/litellm/llms/base_llm/responses/transformation.py @@ -1,7 +1,7 @@ import types from abc import ABC, abstractmethod from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast import httpx @@ -26,6 +26,8 @@ else: LiteLLMLoggingObj = Any BaseLLMException = Any +ReasoningSummaryLevel: TypeAlias = Literal["auto", "concise", "detailed"] + class BaseResponsesAPIConfig(ABC): def __init__(self): @@ -91,6 +93,10 @@ class BaseResponsesAPIConfig(ABC): def get_supported_openai_params(self, model: str) -> list: pass + def auto_reasoning_summary(self) -> ReasoningSummaryLevel: + """The ``reasoning.summary`` LiteLLM's opt-in auto summary asks this provider for.""" + return "detailed" + @abstractmethod def map_openai_params( self, diff --git a/litellm/llms/bedrock_mantle/chat/transformation.py b/litellm/llms/bedrock_mantle/chat/transformation.py index 1c791e994f2..61c7d51a889 100644 --- a/litellm/llms/bedrock_mantle/chat/transformation.py +++ b/litellm/llms/bedrock_mantle/chat/transformation.py @@ -10,19 +10,21 @@ Auth: Bearer token (litellm_params.api_key, BEDROCK_MANTLE_API_KEY, or the BedrockMantleAuthMixin in common_utils. """ -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Iterator, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final import httpx import litellm from litellm._logging import verbose_logger +from litellm.exceptions import UnsupportedParamsError from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock_mantle.common_utils import ( BEDROCK_MANTLE_DEFAULT_REGION, BedrockMantleAuthMixin, ) -from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name +from litellm.llms.openai.chat.gpt_5_transformation import get_effort_level, is_gpt_reasoning_series_name from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams @@ -39,6 +41,51 @@ if TYPE_CHECKING: ) +def _with_chat_reasoning_effort(non_default_params: Mapping[str, object]) -> Mapping[str, object]: + """Chat Completions takes ``reasoning_effort`` as a level, so a Responses-shaped ``{"effort", "summary"}`` + passes only its effort, the way the OpenAI GPT-5 chat config reads it.""" + reasoning_effort: Final = non_default_params.get("reasoning_effort") + if not isinstance(reasoning_effort, dict): + return non_default_params + effort: Final = get_effort_level(reasoning_effort) + rest: Final = {key: value for key, value in non_default_params.items() if key != "reasoning_effort"} + return rest if effort is None else {**rest, "reasoning_effort": effort} + + +def _unsupported_chat_values(non_default_params: Mapping[str, object]) -> tuple[str, ...]: + n: Final = non_default_params.get("n") + reasoning_effort: Final = non_default_params.get("reasoning_effort") + summary: Final = reasoning_effort.get("summary") if isinstance(reasoning_effort, Mapping) else None + return tuple( + value + for value, unsupported in ( + (f"n={n}", isinstance(n, int) and n > 1), + (f"reasoning summary {summary}", summary is not None), + ) + if unsupported + ) + + +def _drops_params(litellm_params: Mapping[str, object]) -> bool: + return litellm.drop_params is True or litellm_params.get("drop_params") is True + + +def _chat_request_params(optional_params: Mapping[str, object], drop_params: bool) -> Mapping[str, object]: + unsupported: Final = _unsupported_chat_values(optional_params) + if unsupported and not drop_params: + raise UnsupportedParamsError( + message=f"Bedrock Mantle Chat Completions does not support {', '.join(unsupported)}. To drop unsupported " + "openai params from the call, set `litellm.drop_params = True`", + llm_provider="bedrock_mantle", + status_code=400, + ) + servable: Final = _with_chat_reasoning_effort(optional_params) + n: Final = servable.get("n") + return MappingProxyType( + {key: value for key, value in servable.items() if not (key == "n" and isinstance(n, int) and n > 1)} + ) + + class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): """ Transformation config for Amazon Bedrock Mantle OpenAI-compatible API. @@ -158,6 +205,28 @@ class BedrockMantleChatConfig(BedrockMantleAuthMixin, OpenAILikeChatConfig): if param != "max_tokens" } + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: OpenAIGPTConfig.transform_request signature + optional_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + litellm_params: dict[str, object], # mutable-ok: OpenAIGPTConfig.transform_request signature + headers: dict[str, str], # mutable-ok: OpenAIGPTConfig.transform_request signature + ) -> dict[str, object]: # mutable-ok: OpenAIGPTConfig.transform_request signature + request_params: Final = dict(_chat_request_params(optional_params, _drops_params(litellm_params))) + return super().transform_request(model, messages, request_params, litellm_params, headers) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + return _chat_request_params( + super().transform_extra_body(extra_body, request, model, litellm_params), _drops_params(litellm_params) + ) + def _supports_reasoning(self, model: str) -> bool: try: return litellm.supports_reasoning(model=model, custom_llm_provider=self.custom_llm_provider) diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index fa0da6a28b4..8a0ce58c8b2 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -24,6 +24,7 @@ import litellm from litellm._logging import verbose_logger from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items +from litellm.llms.base_llm.responses.transformation import ReasoningSummaryLevel from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock_mantle.common_utils import ( @@ -73,6 +74,9 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI def custom_llm_provider(self) -> LlmProviders: return LlmProviders.BEDROCK_MANTLE + def auto_reasoning_summary(self) -> ReasoningSummaryLevel: + return "auto" if self.use_openai_path else super().auto_reasoning_summary() + def get_error_class( self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers ) -> BaseLLMException: diff --git a/litellm/main.py b/litellm/main.py index 3eae64d828e..85d00f28025 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1078,6 +1078,11 @@ def mock_completion( from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site _resolve_openai_api_base as _resolve_openai_api_base, # pyright: ignore[reportPrivateUsage] # noqa: PLC0414 # re-export of moved helper ) +from litellm.responses.bridge_check import ( + bridge_reasoning_effort, + keep_registered_mode, + str_keyed_mapping, +) from litellm.responses.bridge_check import ( # noqa: E402 # re-export at the old definition site responses_api_bridge_check as responses_api_bridge_check, # pyright: ignore[reportUnknownVariableType] # noqa: PLC0414 # re-export of moved function ) @@ -1201,6 +1206,8 @@ def _register_custom_pricing_for_request( pricing overrides (e.g. a zero-cost wildcard) from clobbering built-in pricing used by sibling deployments of the same backend model. Direct SDK calls keep the legacy behavior of registering the shared key with pricing. + Either way the shared key's ``mode`` follows ``keep_registered_mode``, as on + deployment registration, since a call routes by its own ``model_info``. """ entry: Final = _build_custom_pricing_entry( custom_llm_provider=custom_llm_provider, @@ -1210,8 +1217,11 @@ def _register_custom_pricing_for_request( shared_key: Final = f"{custom_llm_provider}/{model}" deployment_id: Final = _get_router_deployment_id(kwargs) if deployment_id is None: - litellm.register_model({shared_key: entry}, persist_across_reloads=False) + litellm.register_model({shared_key: keep_registered_mode(shared_key, entry)}, persist_across_reloads=False) return + shared_entry: Final = keep_registered_mode( + shared_key, CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry) + ) litellm.register_model( {deployment_id: entry}, persist_across_reloads=False, @@ -1220,7 +1230,7 @@ def _register_custom_pricing_for_request( add_to_provider_model_sets=False, ) litellm.register_model( - {shared_key: CustomPricingLiteLLMParams.strip_custom_pricing_fields(entry)}, + {shared_key: shared_entry}, persist_across_reloads=False, warning_display_name=shared_key, custom_llm_provider=custom_llm_provider, @@ -5539,6 +5549,9 @@ def completion( custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, + deployment_model_info=str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ), ) if not _is_claude_tool_target(custom_llm_provider=custom_llm_provider, model=model): @@ -5858,9 +5871,12 @@ def completion( custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, tools=tools, - reasoning_effort=reasoning_effort, + reasoning_effort=bridge_reasoning_effort(reasoning_effort, optional_params.get("extra_body")), reasoning_summary=_reasoning_summary_for_bridge, api_base=api_base, + deployment_model_info=str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ), ) # Use base_model (the true underlying model) for Azure model-type diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6ebb54c0035..13210a7eb1c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10904,7 +10904,6 @@ }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], @@ -10973,7 +10972,6 @@ "output_cost_per_token_batches": 4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], @@ -11005,7 +11003,6 @@ "output_cost_per_token_batches": 4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], diff --git a/litellm/proxy/health_check.py b/litellm/proxy/health_check.py index 14ec69a2059..0d2f8dc9a0c 100644 --- a/litellm/proxy/health_check.py +++ b/litellm/proxy/health_check.py @@ -551,10 +551,13 @@ async def _run_model_health_check(model: dict): model_info, litellm_params ) timeout: Final = model_info.get("health_check_timeout") or HEALTH_CHECK_TIMEOUT_SECONDS + probe_params: Final = ( + {**litellm_params, "model_info": {"mode": model_info["mode"]}} if "mode" in model_info else litellm_params + ) return await run_with_timeout( litellm.ahealth_check( - litellm_params, + probe_params, mode=mode, prompt=DEFAULT_HEALTH_CHECK_PROMPT, input=["test from litellm"], diff --git a/litellm/responses/bridge_check.py b/litellm/responses/bridge_check.py index 2552ee8c880..120a0d7c828 100644 --- a/litellm/responses/bridge_check.py +++ b/litellm/responses/bridge_check.py @@ -13,9 +13,52 @@ from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import OpenAIWebSearchOptions from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager, get_model_info_helper +from litellm.utils import ProviderConfigManager, get_model_info_helper, model_cost_registration_row _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1" +RESPONSES_ONLY_PROVIDERS: Final = frozenset({"chatgpt", "github_copilot"}) +ROUTABLE_MODES: Final = frozenset({"chat", "responses"}) + + +def deployment_routing_mode(deployment_model_info: Mapping[str, object] | None, custom_llm_provider: str) -> str | None: + """The mode a router deployment's own ``model_info`` pins its chat requests to, or ``None`` to defer + to the shared cost-map row. Lets two deployments of one backend model route differently without + either rewriting the shared row the SDK and the other deployments read.""" + if deployment_model_info is None or custom_llm_provider in RESPONSES_ONLY_PROVIDERS: + return None + mode: Final = deployment_model_info.get("mode") + return mode if isinstance(mode, str) and mode in ROUTABLE_MODES else None + + +def keep_registered_mode(backend_key: str, shared_model_info: Mapping[str, object]) -> Mapping[str, object]: + """A deployment's ``chat`` or ``responses`` pin routes only that deployment, so it never replaces the + routing mode registered for its shared backend row, which the SDK and every sibling deployment route by, + and no other mode replaces a registered ``responses`` mode either. + + A ``responses`` pin still marks a row that has no routing mode, since that is how a model missing from + the cost map opts into the Responses API. A mode the row inherits from the built-in entry its registration + falls back to counts as registered, since the first registration writes it whichever deployment that is. A ``chatgpt`` or ``github_copilot`` row takes any mode until it + is ``responses``, since those providers read it by name to pick their endpoint. Elsewhere a ``chat`` pin + never marks a row, so the outcome does not depend on which deployment registers first. A ``mode`` that + is not a string names no mode, so it never reaches the row.""" + deployment_mode: Final = shared_model_info.get("mode") + without_mode: Final = {key: value for key, value in shared_model_info.items() if key != "mode"} + if not isinstance(deployment_mode, str): + return without_mode + registered_mode: Final = model_cost_registration_row( + backend_key, shared_model_info.get("litellm_provider", "") + ).get("mode") + if registered_mode != "responses" and ( + deployment_mode not in ROUTABLE_MODES or backend_key.split("/", 1)[0] in RESPONSES_ONLY_PROVIDERS + ): + return shared_model_info + if registered_mode is None and deployment_mode == "chat": + return without_mode + if registered_mode is None: + return shared_model_info + return {**shared_model_info, "mode": registered_mode} + + _OPENAI_API_HOST: Final = "api.openai.com" @@ -38,6 +81,46 @@ def _resolve_openai_api_base(api_base: str | None) -> str: ) +def _mantle_responses_series_name(model: str) -> str | None: + """The bare OpenAI series name of a Bedrock Mantle model that also serves the Responses API, or ``None``.""" + from litellm.llms.bedrock_mantle.common_utils import mantle_supports_responses, split_mantle_region_prefix + + if not mantle_supports_responses(model, litellm.model_cost): + return None + return split_mantle_region_prefix(model)[1].removeprefix("openai.") + + +def str_keyed_mapping(value: object) -> Mapping[str, object] | None: + """A request field such as a deployment's ``model_info`` when it is a mapping, read without narrowing the + caller's variable (``litellm.completion`` reuses ``model_info`` for pricing registration).""" + if not isinstance(value, Mapping): + return None + return cast( # cast-ok: isinstance only narrows to Mapping[Unknown, Unknown]; request JSON keys are str + Mapping[str, object], value + ) + + +def bridge_reasoning_effort( + reasoning_effort: str | Mapping[str, object] | None, extra_body: object +) -> str | Mapping[str, object] | None: + nested_body: Final = str_keyed_mapping(extra_body) + if reasoning_effort is not None or nested_body is None: + return reasoning_effort + nested_effort: Final = nested_body.get("reasoning_effort") + return nested_effort if isinstance(nested_effort, str) else str_keyed_mapping(nested_effort) + + +def backend_serves_chat_completions(shared_model_info: Mapping[str, object]) -> bool: + """Whether the shared cost-map row lets a deployment pin ``mode: chat``. Responses-only backends + (``gpt-5.1-codex``, ``o3-pro``, ``gpt-5.4-pro``) list only ``/v1/responses`` and must stay bridged, and + a ``responses`` row with no ``supported_endpoints`` (regional Azure codex and pro, Databricks codex) is + read the same way.""" + supported_endpoints: Final = shared_model_info.get("supported_endpoints") + if not isinstance(supported_endpoints, Sequence) or isinstance(supported_endpoints, str): + return shared_model_info.get("mode") != "responses" + return "/v1/chat/completions" in supported_endpoints + + def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | None) -> bool: try: local_model, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) @@ -58,7 +141,8 @@ def responses_api_bridge_check( reasoning_effort: str | Mapping[str, object] | None = None, reasoning_summary: object | None = None, api_base: str | None = None, -) -> tuple[dict, str]: + deployment_model_info: Mapping[str, object] | None = None, +) -> tuple[dict[str, object], str]: model_info: dict[str, object] = {} # Global flag: route ALL OpenAI chat completions through Responses API. @@ -70,21 +154,21 @@ def responses_api_bridge_check( try: model_info = cast( - dict, + dict[str, object], get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider), ) - if model_info.get("mode") is None and model.startswith("responses/"): - model = model.replace("responses/", "") - mode = "responses" - model_info["mode"] = mode - except Exception as e: verbose_logger.debug("Error getting model info: %s", e) - if model.startswith("responses/"): # handle azure models - `azure/responses/` - model = model.replace("responses/", "") - mode = "responses" - model_info["mode"] = mode + explicit_responses_prefix: Final = model.startswith("responses/") + pinned_mode: Final = ( + None if explicit_responses_prefix else deployment_routing_mode(deployment_model_info, custom_llm_provider) + ) + if explicit_responses_prefix or pinned_mode == "responses": + model = model.replace("responses/", "") + model_info["mode"] = "responses" + elif pinned_mode == "chat" and backend_serves_chat_completions(model_info): + model_info["mode"] = "chat" if web_search_options is not None and custom_llm_provider == "xai": model_info["mode"] = "responses" @@ -112,6 +196,9 @@ def responses_api_bridge_check( # an explicit effort with function tools is rejected from gpt-5.6 on, and the unset # effort only from gpt-6 on (gpt-5.6 serves tools with reasoning silently off), so the # azure_ai gate keys on those measured boundaries instead of gpt-5.4+. + # - Bedrock Mantle serves the same API and enforces the gpt-5.4+ rule even with the effort + # unset (measured on gpt-5.6-terra), so a Mantle row that also serves Responses follows + # these rules, which keeps a deployment pinned to ``mode: chat`` from a 400 on them. # - Older GPT-5 names (e.g. ``gpt-5``, ``gpt-5.1``): bridge only when a reasoning # summary alias is present with ``reasoning_effort`` (tools alone stay on chat). has_function_tool: Final = any( @@ -140,8 +227,21 @@ def responses_api_bridge_check( on_foundry_openai_endpoint: Final = custom_llm_provider == "azure_ai" and azure_ai_supports_native_responses( model, api_base ) + mantle_series_name: Final = ( + _mantle_responses_series_name(model) if custom_llm_provider == "bedrock_mantle" else None + ) + on_mantle_responses_endpoint: Final = mantle_series_name is not None + summary_requested: Final = reasoning_summary is not None or ( + on_mantle_responses_endpoint + and isinstance(reasoning_effort, Mapping) + and reasoning_effort.get("summary") is not None + ) + gpt_series_name: Final = mantle_series_name or model on_constraint_enforcing_endpoint: Final = ( - custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base) + custom_llm_provider == "azure" + or on_mantle_responses_endpoint + or resolved_api_base == "" + or _is_openai_backed_api_base(resolved_api_base) ) chat_rejects_function_tools: Final = ( has_function_tool @@ -150,20 +250,21 @@ def responses_api_bridge_check( foundry_chat_rejects_function_tools_while_reasoning(model, reasoning_effort) if on_foundry_openai_endpoint else ( - OpenAIGPT5Config.is_model_gpt_5_4_plus_model(model) + OpenAIGPT5Config.is_model_gpt_5_4_plus_model(gpt_series_name) and (reasoning_effort is not None or on_constraint_enforcing_endpoint) ) ) ) if ( - (custom_llm_provider in ("openai", "azure") or on_foundry_openai_endpoint) + (custom_llm_provider in ("openai", "azure") or on_foundry_openai_endpoint or on_mantle_responses_endpoint) and model_info.get("mode") != "responses" - and OpenAIGPT5Config.is_model_gpt_5_model(model) - and not OpenAIGPT5Config.is_model_gpt_5_search_model(model) + and OpenAIGPT5Config.is_model_gpt_5_model(gpt_series_name) + and not OpenAIGPT5Config.is_model_gpt_5_search_model(gpt_series_name) and ( - (reasoning_effort is not None and reasoning_summary is not None) + (reasoning_effort is not None and summary_requested) or chat_rejects_function_tools or (carries_tool_search and on_constraint_enforcing_endpoint) + or (web_search_options is not None and on_mantle_responses_endpoint) ) ): model_info["mode"] = "responses" diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 35d4b8bf1ba..842230699b7 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -762,8 +762,8 @@ class LiteLLMCompletionResponsesConfig: """ Whether ``litellm.completion`` will route this model back onto the Responses API. - Delegates to the same check ``litellm.completion`` itself runs, and is asked with the - params this transform is about to emit, so the two cannot reach different answers. + Delegates to the same check ``litellm.completion`` itself runs, asked with the params this + transform is about to emit. A deployment's own ``model_info.mode`` pin is not passed here. """ from litellm.main import responses_api_bridge_check diff --git a/litellm/responses/main.py b/litellm/responses/main.py index d32d48c51ca..ec1786b529a 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -1555,7 +1555,7 @@ def responses( ) reasoning_effort: Final[object] = local_vars.get("reasoning_effort") request_reasoning: Final = ( - LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort) + LiteLLMResponsesTransformationHandler().map_reasoning_effort(reasoning_effort, model, custom_llm_provider) if current_reasoning is None and reasoning_effort is not None else current_reasoning ) diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index b16fe72b281..8e5e1c1cfee 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -10,7 +10,7 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path -from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig +from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, OutputTokensDetails, @@ -1294,3 +1294,19 @@ class ResponseAPILoggingUtils: return chat_usage _transform_response_api_usage_to_chat_usage = transform_response_api_usage_to_chat_usage + + +def auto_reasoning_summary(model: str, custom_llm_provider: object) -> ReasoningSummaryLevel: + """The summary level the auto summary injects for a target, read from its Responses API config.""" + from litellm.utils import ProviderConfigManager + + try: + provider: Final = ( + custom_llm_provider if isinstance(custom_llm_provider, str) else litellm.get_llm_provider(model=model)[1] + ) + except litellm.exceptions.BadRequestError: + return "detailed" + responses_config: Final = ProviderConfigManager.get_provider_responses_api_config( + provider, model.removeprefix(f"{provider}/") + ) + return "detailed" if responses_config is None else responses_config.auto_reasoning_summary() diff --git a/litellm/router.py b/litellm/router.py index 8d77fd0dac5..e363f41ccde 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -139,6 +139,7 @@ from litellm.llms.openai_like.model_info import ( MODEL_INFO_REFRESH_SECONDS, get_openai_compatible_model_info, ) +from litellm.responses.bridge_check import keep_registered_mode from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.complexity_router.context_compaction import ( @@ -11010,7 +11011,6 @@ class Router: ## OLD MODEL REGISTRATION ## Kept to prevent breaking changes backend_keys: Final = Router._backend_cost_map_keys(model=model, custom_llm_provider=custom_llm_provider) - backend_key: Final = backend_keys[0] # For the shared backend key, keep only cost-map schema fields # (minus custom pricing) so that one deployment's pricing overrides @@ -11019,34 +11019,8 @@ class Router: # name. Each deployment's full model_info is already stored under # its unique model_id above. shared_model_info: Final = shared_backend_model_info(model_info) - existing_shared_mode: Final = (cast(dict | None, litellm.model_cost.get(backend_key, {})) or {}).get("mode") - deployment_mode: Final = shared_model_info.get("mode") - # Keep the built-in bridge mode stable for shared backend keys. - # Multiple aliases can point at the same provider/model backend, - # but their deployment-level overrides should not downgrade the - # backend from responses -> chat via last-write-wins registration. - # Only preserve in that specific direction so legitimate upgrades - # (e.g. chat -> responses) and unrelated mode changes still apply, - # and so a missing deployment mode does not silently clear the - # existing shared backend mode. - is_responses_to_chat_downgrade: Final = existing_shared_mode == "responses" and deployment_mode == "chat" - would_clear_existing_mode: Final = existing_shared_mode is not None and deployment_mode is None - if is_responses_to_chat_downgrade or would_clear_existing_mode: - if deployment_mode is not None: - verbose_router_logger.warning( - "Router: preserving existing mode=%s for shared backend " - "key %s instead of the deployment-specified mode=%s " - "(prevents alias registration from downgrading the " - "shared backend mode).", - existing_shared_mode, - backend_key, - deployment_mode, - ) - shared_model_info["mode"] = existing_shared_mode - - # Always register the (possibly mode-preserved) shared backend info. litellm.register_model( - model_cost={_key: shared_model_info for _key in backend_keys}, + model_cost={_key: keep_registered_mode(_key, shared_model_info) for _key in backend_keys}, persist_across_reloads=False, ) diff --git a/litellm/utils.py b/litellm/utils.py index b37827a3b13..d6bdee4697b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3520,19 +3520,33 @@ def _get_builtin_model_info_for_registration(model: str, custom_llm_provider: st ) +def _skips_builtin_registration_lookup(key: str, provider: object) -> bool: + return provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO or any( + key.startswith(f"{p}/") for p in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO + ) + + @dataclass(frozen=True, slots=True) class _RegistrationWrite: key: str entry: Mapping[str, object] +def model_cost_registration_row(key: str, provider: object) -> Mapping[str, object]: + """The row a ``register_model`` write of a ``key`` entry with this ``litellm_provider`` merges onto: the + ``litellm.model_cost`` row it resolves to (``openai/o3-pro`` lands on ``o3-pro``), else the row it seeds ``key`` + with from the built-in entry it falls back to (an OpenAI fine-tune takes its base model's row).""" + base: Final = _registration_base( + key, {"litellm_provider": provider}, custom_llm_provider=None, warning_display_name=None + ) + return _builtin_cost_map().get(base.key) or base.entry + + def _registration_base( key: str, value: Mapping[str, object], custom_llm_provider: str | None, warning_display_name: str | None ) -> _RegistrationWrite: provider: Final = str(value.get("litellm_provider") or "") - if provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO or any( - key.startswith(f"{p}/") for p in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO - ): + if _skips_builtin_registration_lookup(key, provider): return _RegistrationWrite(key=key, entry=dict(_builtin_cost_map().get(key, {}))) builtin_model_info: Final = _get_builtin_model_info_for_registration( model=key, custom_llm_provider=custom_llm_provider diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6ebb54c0035..13210a7eb1c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10904,7 +10904,6 @@ }, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], @@ -10973,7 +10972,6 @@ "output_cost_per_token_batches": 4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], @@ -11005,7 +11003,6 @@ "output_cost_per_token_batches": 4e-05, "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", "supported_endpoints": [ - "/v1/chat/completions", "/v1/batch", "/v1/responses" ], diff --git a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py index 9b07cce2f3a..1e599103a27 100644 --- a/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_output_limit_wire.py @@ -250,6 +250,7 @@ def _deployment( *, base: str = _OPENAI_BASE, api_key: str = _API_KEY, + model_info: Mapping[str, JsonValue] | None = _CHAT_INFO, **extra: JsonValue, ) -> str: reasoning: Final[Mapping[str, JsonValue]] = ( @@ -260,7 +261,7 @@ def _deployment( custom_llm_provider="bedrock_mantle", api_base=wire.url + base, api_key=api_key, - model_info=_CHAT_INFO, + model_info=model_info, **reasoning, **extra, ) @@ -429,7 +430,7 @@ def test_stream_with_tools_carries_the_deployment_limit_as_max_completion_tokens text: Final = _stream_text( gateway, "/v1/chat/completions", - _chat_body(model, marker, stream=True, stream_options={"include_usage": True}, tools=_TOOLS), + _chat_body(model, marker, stream=True, stream_options={"include_usage": True}, tools=list(_TOOLS)), ) assert f"marker-{marker}" in text and text.rstrip().endswith("data: [DONE]"), text (attempt,) = _attempts(wire) @@ -471,6 +472,22 @@ async def test_async_anthropic_sdk_messages_stream_reaches_mantle_as_max_complet _assert_logged(message.id, model, "success") +def test_a_chat_mode_deployment_leaves_its_unpinned_sibling_on_the_responses_api(gateway: Gateway) -> None: + pinned_marker: Final = uuid.uuid4().hex + sibling_marker: Final = uuid.uuid4().hex + with wire_server(_mantle) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _GPT56) + sibling: Final = _deployment(scenario, wire, _GPT56, model_info=None) + for model, marker in ((pinned, pinned_marker), (sibling, sibling_marker)): + response = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker)) + assert response.status_code == 200, response.text + assert _answer(marker) in response.text, response.text + pinned_attempt, sibling_attempt = _attempts(wire) + assert (pinned_attempt.target, _marker_in(pinned_attempt)) == (_CHAT_TARGET, pinned_marker), pinned_attempt + assert sibling_attempt.target in _RESPONSES_TARGETS, sibling_attempt + assert _marker_in(sibling_attempt) == sibling_marker, sibling_attempt + + def test_responses_max_output_tokens_is_forwarded_natively_on_both_legs(gateway: Gateway) -> None: unary: Final = uuid.uuid4().hex streamed: Final = uuid.uuid4().hex @@ -500,7 +517,6 @@ def test_responses_max_output_tokens_is_forwarded_natively_on_both_legs(gateway: _assert_logged(_managed_response_id(streamed_issued), model, "success") -@pytest.mark.skip(reason="unrealistic input") def test_region_prefixed_deployment_on_a_v1_base_sends_the_bare_model_with_max_completion_tokens( gateway: Gateway, ) -> None: diff --git a/tests/integration/routing/test_deployment_mode_routing_wire.py b/tests/integration/routing/test_deployment_mode_routing_wire.py new file mode 100644 index 00000000000..65e64883b18 --- /dev/null +++ b/tests/integration/routing/test_deployment_mode_routing_wire.py @@ -0,0 +1,1037 @@ +import asyncio +import itertools +import json +import re +import threading +import uuid +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import urlsplit + +import anthropic +import httpx +import openai +import psutil +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.responses_vendor import same_response +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_API_KEY: Final = "synthetic-routing-bearer" +_MANTLE_GPT: Final = "bedrock_mantle/openai.gpt-5.6-terra" +_MANTLE_OSS: Final = "bedrock_mantle/openai.gpt-oss-120b" +_OPENAI_GPT: Final = "openai/gpt-5.4" +_PINNED: Final = "routing-chat-pinned" +_SIBLING: Final = "routing-unpinned-sibling" +_NO_RETRIES: Final[Mapping[str, JsonValue]] = MappingProxyType({"num_retries": 0}) +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})") +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_SPEND_BY_MODEL_GROUP: Final = 'SELECT request_id, status, model_group FROM "LiteLLM_SpendLogs" WHERE model_group = %s' +_N_REJECTED_BY_MANTLE: Final = "'n' is not supported for values greater than 1" +_FUNCTION_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Current weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, + } +) + +Api = Literal["chat", "responses"] +Client = Literal["openai-sdk", "openai-sdk-async-stream", "anthropic-sdk", "anthropic-sdk-async-stream", "httpx-stream"] +Endpoint = Literal["chat", "messages", "responses"] + + +@dataclass(frozen=True, slots=True) +class _Attempt: + api: Api + marker: str + body: Mapping[str, JsonValue] + + +@dataclass(frozen=True, slots=True) +class _Answer: + status: int + identity: str + call_id: str + text: str + + +@dataclass(frozen=True, slots=True) +class _Call: + model: str + endpoint: Endpoint + stream: bool + marker: str + + +def _question(marker: str) -> str: + return f"Question marker-{marker}" + + +def _answer(marker: str) -> str: + return f"answer marker-{marker}" + + +def _marker_in(text: str) -> str: + found: Final = _MARKER.search(text) + assert found is not None, text + return found.group(1) + + +def _api(target: str) -> Api: + if target.endswith("/chat/completions"): + return "chat" + assert target.endswith("/responses"), target + return "responses" + + +def _error(status: int, message: str, kind: str) -> Reply: + return Reply(status=status, body=json.dumps({"error": {"message": message, "type": kind, "code": None}}).encode()) + + +def _upstream(down: threading.Event | None = None) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.method != "POST": + return Reply(body=b'{"object": "list", "data": []}') + if request.headers.get("authorization") != f"Bearer {_API_KEY}": + return _error(401, "Invalid bearer token.", "authentication_error") + if down is not None and down.is_set(): + return _error(503, "The server is overloaded.", "server_error") + body: Final = _JSON_OBJECT.validate_json(request.body) + marker: Final = _marker_in(request.body.decode()) + stream: Final = body.get("stream") is True + backend: Final = string_value(body["model"]) + if _api(request.target) == "responses": + return responses_reply(f"resp_{marker}", backend, _answer(marker), stream=stream) + n: Final = body.get("n") + if isinstance(n, int) and n > 1: + return _error(400, _N_REJECTED_BY_MANTLE, "invalid_request_error") + return chat_reply(f"chatcmpl-{marker}", backend, _answer(marker), stream=stream) + + return respond + + +def _is_probe(request: Request) -> bool: + return request.method == "POST" and _MARKER.search(request.body.decode()) is None + + +def _attempts(wire: Wire, *, skip_probes: bool = False) -> tuple[_Attempt, ...]: + return tuple( + _Attempt(_api(request.target), _marker_in(request.body.decode()), _JSON_OBJECT.validate_json(request.body)) + for request in wire.drain() + if request.method == "POST" and not (skip_probes and _is_probe(request)) + ) + + +def _routes(attempts: tuple[_Attempt, ...]) -> Mapping[str, tuple[Api, ...]]: + markers: Final = frozenset(attempt.marker for attempt in attempts) + return {marker: tuple(attempt.api for attempt in attempts if attempt.marker == marker) for marker in markers} + + +def _api_base(wire: Wire, backend: str) -> str: + return wire.url + ("/openai/v1" if backend.startswith("bedrock_mantle/") else "/v1") + + +def _deployment( + scenario: Scenario, + wire: Wire, + backend: str, + mode: str | None, + *, + provider_pin: bool = True, + api_key: str = _API_KEY, +) -> str: + provider: Final[Mapping[str, JsonValue]] = {"custom_llm_provider": backend.split("/", 1)[0]} if provider_pin else {} + return scenario.model( + model=backend, + api_base=_api_base(wire, backend), + api_key=api_key, + model_info=None if mode is None else {"mode": mode}, + **provider, + ) + + +def _wildcard(scenario: Scenario, wire: Wire, mode: str | None) -> str: + prefix: Final = f"wild-{uuid.uuid4().hex}" + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": f"{prefix}/*", + "litellm_params": {"model": "bedrock_mantle/*", "api_base": wire.url + "/openai/v1", "api_key": _API_KEY}, + "model_info": {} if mode is None else {"mode": mode}, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + return f"{prefix}/{_MANTLE_GPT.removeprefix('bedrock_mantle/')}" + + +def _raw_model_info_deployment(scenario: Scenario, wire: Wire, raw_model_info: str) -> str: + name: Final = f"integration-{uuid.uuid4().hex}" + params: Final = json.dumps( + { + "model": _MANTLE_GPT, + "custom_llm_provider": "bedrock_mantle", + "api_base": wire.url + "/openai/v1", + "api_key": _API_KEY, + } + ) + response: Final = scenario.gateway.client.post( + "/model/new", + content=f'{{"model_name": "{name}", "litellm_params": {params}, "model_info": {raw_model_info}}}', + headers={"Authorization": f"Bearer {scenario.gateway.key}", "Content-Type": "application/json"}, + ) + assert response.status_code == 200, response.text + identity: Final = string_value(object_value(_JSON_OBJECT.validate_json(response.content)["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return name + + +def _proxy_url(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _messages(marker: str) -> list[JsonValue]: + return [{"role": "user", "content": _question(marker)}] + + +def _identity(endpoint: Endpoint, stream: bool, text: str) -> str: + if not stream: + return string_value(_JSON_OBJECT.validate_json(text)["id"]) + first: Final = _JSON_OBJECT.validate_json( + next(line for line in text.splitlines() if line.startswith("data: {")).removeprefix("data: ") + ) + match endpoint: + case "chat": + return string_value(first["id"]) + case "messages": + return string_value(object_value(first["message"])["id"]) + case "responses": + return string_value(object_value(first["response"])["id"]) + + +def _body(call: _Call, extra: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + match call.endpoint: + case "chat": + return {"model": call.model, "messages": _messages(call.marker), "stream": call.stream, **extra} + case "messages": + return {"model": call.model, "max_tokens": 64, "messages": _messages(call.marker), "stream": call.stream} + case "responses": + return {"model": call.model, "input": _question(call.marker), "stream": call.stream, **extra} + + +def _path(endpoint: Endpoint) -> str: + match endpoint: + case "chat": + return "/v1/chat/completions" + case "messages": + return "/v1/messages" + case "responses": + return "/v1/responses" + + +async def _send(client: httpx.AsyncClient, key: str, call: _Call, extra: Mapping[str, JsonValue]) -> _Answer: + async with client.stream( + "POST", + _path(call.endpoint), + json=_body(call, extra), + headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"}, + ) as response: + text: Final = (await response.aread()).decode() + identity: Final = _identity(call.endpoint, call.stream, text) if response.status_code == 200 else "" + call_id: Final = response.headers["x-litellm-call-id"] if "x-litellm-call-id" in response.headers else "" + return _Answer(response.status_code, identity, call_id, text) + + +async def _http(gateway: Gateway, call: _Call, extra: Mapping[str, JsonValue] = _NO_RETRIES) -> _Answer: + async with httpx.AsyncClient(base_url=_proxy_url(gateway), timeout=60, trust_env=False) as client: + return await _send(client, gateway.key, call, extra) + + +async def _burst( + base_url: str, key: str, calls: tuple[_Call, ...], *, tolerate_drops: bool = False +) -> tuple[_Answer | None, ...]: + async with httpx.AsyncClient(base_url=base_url, timeout=90, trust_env=False) as client: + results: Final = await asyncio.gather( + *(_send(client, key, call, _NO_RETRIES) for call in calls), return_exceptions=tolerate_drops + ) + for result in results: + assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result) + return tuple(result if isinstance(result, _Answer) else None for result in results) + + +def _openai_sdk(gateway: Gateway, model: str, marker: str) -> _Answer: + with openai.OpenAI(base_url=f"{_proxy_url(gateway)}/v1", api_key=gateway.key, max_retries=0) as sdk: + completion: Final = sdk.chat.completions.create( + model=model, messages=[{"role": "user", "content": _question(marker)}] + ) + return _Answer(200, completion.id, "", completion.choices[0].message.content or "") + + +async def _openai_sdk_async_stream(gateway: Gateway, model: str, marker: str) -> _Answer: + async with openai.AsyncOpenAI(base_url=f"{_proxy_url(gateway)}/v1", api_key=gateway.key, max_retries=0) as sdk: + stream: Final = await sdk.chat.completions.create( + model=model, messages=[{"role": "user", "content": _question(marker)}], stream=True + ) + chunks: Final = [chunk async for chunk in stream] + (identity,) = {chunk.id for chunk in chunks} + return _Answer( + 200, identity, "", "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) + ) + + +def _anthropic_sdk(gateway: Gateway, model: str, marker: str) -> _Answer: + with anthropic.Anthropic(base_url=_proxy_url(gateway), api_key=gateway.key, max_retries=0) as sdk: + message: Final = sdk.messages.create( + model=model, max_tokens=64, messages=[{"role": "user", "content": _question(marker)}] + ) + return _Answer(200, message.id, "", "".join(block.text for block in message.content if block.type == "text")) + + +async def _anthropic_sdk_async_stream(gateway: Gateway, model: str, marker: str) -> _Answer: + async with anthropic.AsyncAnthropic(base_url=_proxy_url(gateway), api_key=gateway.key, max_retries=0) as sdk: + async with sdk.messages.stream( + model=model, max_tokens=64, messages=[{"role": "user", "content": _question(marker)}] + ) as stream: + pieces: Final = [piece async for piece in stream.text_stream] + message: Final = await stream.get_final_message() + return _Answer(200, message.id, "", "".join(pieces)) + + +async def _ask(gateway: Gateway, client: Client, model: str, marker: str) -> _Answer: + match client: + case "openai-sdk": + return _openai_sdk(gateway, model, marker) + case "openai-sdk-async-stream": + return await _openai_sdk_async_stream(gateway, model, marker) + case "anthropic-sdk": + return _anthropic_sdk(gateway, model, marker) + case "anthropic-sdk-async-stream": + return await _anthropic_sdk_async_stream(gateway, model, marker) + case "httpx-stream": + return await _http(gateway, _Call(model, "chat", True, marker)) + + +def _logged_ids(answer: _Answer) -> tuple[str, ...]: + markers: Final = frozenset(_MARKER.findall(answer.text)) + upstream: Final = (*(f"resp_{marker}" for marker in markers), *(f"chatcmpl-{marker}" for marker in markers)) + return tuple(shown for shown in (answer.identity, answer.call_id, *upstream) if shown) + + +def _logs_answer(request_id: JsonValue, answer: _Answer) -> bool: + return any(same_response(string_value(request_id), shown) for shown in _logged_ids(answer)) + + +def _logged_rows(answer: _Answer, model: str) -> list[dict[str, JsonValue]]: + return [row for row in read_rows(_SPEND_BY_MODEL_GROUP, (model,)) if _logs_answer(row["request_id"], answer)] + + +def _assert_logged(answer: _Answer, model: str, status: str) -> None: + rows: Final = eventually(lambda: _logged_rows(answer, model), lambda found: len(found) >= 1, seconds=70) + assert len(rows) == 1, rows + assert (rows[0]["status"], rows[0]["model_group"]) == (status, model), rows + + +def _assert_answered(answer: _Answer | None, marker: str) -> _Answer: + assert answer is not None, marker + assert answer.status == 200, answer.text + assert _answer(marker) in answer.text, answer.text + assert set(_MARKER.findall(answer.text)) == {marker}, answer.text + return answer + + +def _waves(model: str, count: int) -> tuple[_Call, ...]: + endpoints: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") + return tuple( + _Call(model, endpoints[index % len(endpoints)], index % 2 == 0, uuid.uuid4().hex) for index in range(count) + ) + + +def _expected_route(call: _Call, pinned: str) -> Api: + return "chat" if call.model == pinned and call.endpoint != "responses" else "responses" + + +@pytest.mark.parametrize( + "client", ("openai-sdk", "openai-sdk-async-stream", "anthropic-sdk", "anthropic-sdk-async-stream", "httpx-stream") +) +async def test_a_chat_pin_leaves_its_unpinned_sibling_on_responses_on_every_client( + gateway: Gateway, client: Client +) -> None: + pinned_marker: Final = uuid.uuid4().hex + sibling_marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + pinned_answer: Final = _assert_answered(await _ask(gateway, client, pinned, pinned_marker), pinned_marker) + sibling_answer: Final = _assert_answered(await _ask(gateway, client, sibling, sibling_marker), sibling_marker) + assert _routes(_attempts(wire)) == {pinned_marker: ("chat",), sibling_marker: ("responses",)} + _assert_logged(pinned_answer, pinned, "success") + _assert_logged(sibling_answer, sibling, "success") + + +@pytest.mark.parametrize("backend", (_MANTLE_OSS, _OPENAI_GPT)) +async def test_a_responses_pin_leaves_its_unpinned_sibling_on_chat(gateway: Gateway, backend: str) -> None: + pinned_marker: Final = uuid.uuid4().hex + sibling_marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, backend, "responses") + sibling: Final = _deployment(scenario, wire, backend, None) + pinned_answer: Final = _assert_answered( + await _http(gateway, _Call(pinned, "chat", False, pinned_marker)), pinned_marker + ) + sibling_answer: Final = _assert_answered( + await _http(gateway, _Call(sibling, "chat", False, sibling_marker)), sibling_marker + ) + assert _routes(_attempts(wire)) == {pinned_marker: ("responses",), sibling_marker: ("chat",)} + _assert_logged(pinned_answer, pinned, "success") + _assert_logged(sibling_answer, sibling, "success") + + +@pytest.mark.parametrize("stream", (False, True)) +async def test_the_responses_endpoint_serves_a_chat_pinned_deployment_natively(gateway: Gateway, stream: bool) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + answer: Final = _assert_answered(await _http(gateway, _Call(pinned, "responses", stream, marker)), marker) + assert _routes(_attempts(wire)) == {marker: ("responses",)} + _assert_logged(answer, pinned, "success") + + +async def test_a_wildcard_route_keeps_the_registered_mode_beside_a_chat_pin(gateway: Gateway) -> None: + pinned_marker: Final = uuid.uuid4().hex + wildcard_marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + wildcard: Final = _wildcard(scenario, wire, None) + _assert_answered(await _http(gateway, _Call(pinned, "chat", False, pinned_marker)), pinned_marker) + _assert_answered(await _http(gateway, _Call(wildcard, "chat", False, wildcard_marker)), wildcard_marker) + assert _routes(_attempts(wire)) == {pinned_marker: ("chat",), wildcard_marker: ("responses",)} + + +@pytest.mark.parametrize("shape", ("wildcard", "no-provider-pin")) +async def test_a_chat_pin_without_a_provider_pin_gets_chat_completions(gateway: Gateway, shape: str) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + model: Final = ( + _wildcard(scenario, wire, "chat") + if shape == "wildcard" + else _deployment(scenario, wire, _MANTLE_GPT, "chat", provider_pin=False) + ) + _assert_answered(await _http(gateway, _Call(model, "chat", False, marker)), marker) + assert _routes(_attempts(wire)) == {marker: ("chat",)} + + +async def test_an_explicit_responses_prefix_wins_over_a_chat_pin(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/responses/gpt-5.4", api_base=wire.url + "/v1", api_key=_API_KEY, model_info={"mode": "chat"} + ) + answer: Final = _assert_answered(await _http(gateway, _Call(model, "chat", False, marker)), marker) + (attempt,) = _attempts(wire) + assert (attempt.api, attempt.body["model"]) == ("responses", "gpt-5.4"), attempt + _assert_logged(answer, model, "success") + + +_SUMMARY_REASONING: Final[Mapping[str, JsonValue]] = MappingProxyType({"effort": "low", "summary": "auto"}) + + +@pytest.mark.parametrize( + ("extra", "reasoning"), + ( + ({"tools": [dict(_FUNCTION_TOOL)]}, None), + ({"reasoning_effort": dict(_SUMMARY_REASONING)}, _SUMMARY_REASONING), + ({"extra_body": {"reasoning_effort": dict(_SUMMARY_REASONING)}}, _SUMMARY_REASONING), + ({"web_search_options": {"search_context_size": "low"}}, None), + ), + ids=("function-tools", "reasoning-summary", "extra-body-reasoning-summary", "web-search"), +) +async def test_a_mantle_chat_pin_sends_what_chat_cannot_serve_to_responses( + gateway: Gateway, extra: Mapping[str, JsonValue], reasoning: Mapping[str, JsonValue] | None +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + answer: Final = _assert_answered( + await _http(gateway, _Call(pinned, "chat", False, marker), {**_NO_RETRIES, **extra}), marker + ) + (attempt,) = _attempts(wire) + assert attempt.api == "responses", attempt + assert reasoning is None or attempt.body["reasoning"] == reasoning, attempt.body + _assert_logged(answer, pinned, "success") + + +def _placed(placement: str, params: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: + return params if placement == "top-level" else {"extra_body": dict(params)} + + +async def _messages_with_thinking( + gateway: Gateway, model: str, marker: str, thinking: Mapping[str, JsonValue] +) -> _Answer: + async with httpx.AsyncClient(base_url=_proxy_url(gateway), timeout=60, trust_env=False) as client: + response: Final = await client.post( + "/v1/messages", + json={"model": model, "max_tokens": 2048, "thinking": dict(thinking), "messages": _messages(marker)}, + headers={"Authorization": f"Bearer {gateway.key}", "anthropic-version": "2023-06-01"}, + ) + identity: Final = _identity("messages", False, response.text) if response.status_code == 200 else "" + return _Answer(response.status_code, identity, response.headers.get("x-litellm-call-id", ""), response.text) + + +@pytest.mark.parametrize(("summary", "route"), ((None, "chat"), ("auto", "responses")), ids=("plain", "summary")) +async def test_a_mantle_chat_pin_moves_messages_thinking_to_responses_only_when_it_asks_for_a_summary( + gateway: Gateway, summary: str | None, route: Api +) -> None: + marker: Final = uuid.uuid4().hex + thinking: Final[Mapping[str, JsonValue]] = {"type": "enabled", "budget_tokens": 1024} | ( + {} if summary is None else {"summary": summary} + ) + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + answer: Final = _assert_answered(await _messages_with_thinking(gateway, pinned, marker, thinking), marker) + (attempt,) = _attempts(wire) + assert attempt.api == route, attempt + match route: + case "chat": + assert "summary" not in json.dumps(attempt.body), attempt.body + case "responses": + assert object_value(attempt.body["reasoning"])["summary"] == summary, attempt.body + _assert_logged(answer, pinned, "success") + + +def _sent_effort(attempt: _Attempt) -> JsonValue: + match attempt.api: + case "chat": + return attempt.body.get("reasoning_effort") + case "responses": + return object_value(attempt.body["reasoning"]).get("effort") + + +@pytest.mark.parametrize( + ("backend", "mode", "effort", "route"), + ((_OPENAI_GPT, None, "low", "responses"), (_MANTLE_GPT, "chat", "none", "chat")), + ids=("openai-low", "mantle-chat-pin-none"), +) +@pytest.mark.parametrize("placement", ("top-level", "extra-body")) +async def test_an_effort_inside_extra_body_routes_function_tools_like_a_top_level_one( + gateway: Gateway, backend: str, mode: str | None, effort: str, route: Api, placement: str +) -> None: + marker: Final = uuid.uuid4().hex + extra: Final[Mapping[str, JsonValue]] = { + **_NO_RETRIES, + "tools": [dict(_FUNCTION_TOOL)], + **_placed(placement, {"reasoning_effort": effort}), + } + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, backend, mode) + answer: Final = _assert_answered(await _http(gateway, _Call(model, "chat", False, marker), extra), marker) + (attempt,) = _attempts(wire) + assert (attempt.api, _sent_effort(attempt)) == (route, effort), attempt + _assert_logged(answer, model, "success") + + +_AUTO_SUMMARY_LEVELS: Final[Mapping[str, str]] = MappingProxyType( + {"auto-summary-mantle": "auto", "auto-summary-openai": "detailed"} +) +_PLAIN_THINKING: Final[Mapping[str, JsonValue]] = MappingProxyType({"type": "enabled", "budget_tokens": 1024}) + + +def _auto_summary_config(wire: Wire, tmp_path: Path) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + mantle: Final[dict[str, JsonValue]] = { + "model": _MANTLE_GPT, + "custom_llm_provider": "bedrock_mantle", + "api_base": wire.url + "/openai/v1", + "api_key": _API_KEY, + } + openai_codex: Final[dict[str, JsonValue]] = { + "model": "openai/gpt-5.3-codex", + "api_base": wire.url + "/v1", + "api_key": _API_KEY, + } + config: Final = { + **base, + "model_list": [ + {"model_name": "auto-summary-mantle", "litellm_params": mantle}, + {"model_name": "auto-summary-openai", "litellm_params": openai_codex}, + ], + "router_settings": {**object_value(base["router_settings"]), "num_retries": 0}, + } + path: Final = tmp_path / "deployment-mode-routing-auto-summary.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +async def _ask_for_reasoning(gateway: Gateway, call: _Call) -> _Answer: + if call.endpoint == "messages": + return await _messages_with_thinking(gateway, call.model, call.marker, _PLAIN_THINKING) + return await _http(gateway, call, {**_NO_RETRIES, "reasoning_effort": "low"}) + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +async def test_the_auto_summary_asks_each_target_for_a_level_it_accepts_on_every_endpoint( + gateway: Gateway, tmp_path: Path +) -> None: + endpoints: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") + calls: Final = tuple( + _Call(model, endpoint, False, uuid.uuid4().hex) + for model, endpoint in itertools.product(_AUTO_SUMMARY_LEVELS, endpoints) + ) + with wire_server(_upstream()) as wire: + config: Final = _auto_summary_config(wire, tmp_path) + with owned_proxy_process(gateway, tmp_path, {"LITELLM_REASONING_AUTO_SUMMARY": "true"}, config=config) as owned: + for call in calls: + _assert_answered(await _ask_for_reasoning(owned.gateway, call), call.marker) + sent: Final = { + attempt.marker: (attempt.api, object_value(attempt.body["reasoning"]).get("summary")) + for attempt in _attempts(wire) + } + assert sent == {call.marker: ("responses", _AUTO_SUMMARY_LEVELS[call.model]) for call in calls} + + +def _health_upstream(request: Request) -> Reply: + if request.method != "POST": + return Reply(body=b'{"object": "list", "data": []}') + backend: Final = string_value(_JSON_OBJECT.validate_json(request.body)["model"]) + if _api(request.target) == "responses": + return responses_reply("resp_health", backend, "ok", stream=False) + return chat_reply("chatcmpl-health", backend, "ok", stream=False) + + +@pytest.mark.parametrize(("mode", "route"), (("chat", "chat"), (None, "responses")), ids=("chat-pin", "unpinned")) +def test_a_health_check_probes_the_api_its_deployment_sends_chat_to( + gateway: Gateway, mode: str | None, route: Api +) -> None: + with wire_server(_health_upstream) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, _MANTLE_GPT, mode) + response: Final = gateway.request("GET", "/health", params={"model": model}) + probed: Final = tuple(_api(request.target) for request in wire.drain() if request.method == "POST") + assert response.status_code == 200, response.text + assert _JSON_OBJECT.validate_json(response.content)["healthy_count"] == 1, response.text + assert probed == (route,), probed + + +_INPUT_PRICE: Final = 0.001 +_OUTPUT_PRICE: Final = 0.002 +_SPEND_OF_MODEL_GROUP: Final = 'SELECT request_id, spend FROM "LiteLLM_SpendLogs" WHERE model_group = %s' + + +def _priced_chat_pin(scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model=_MANTLE_GPT, + custom_llm_provider="bedrock_mantle", + api_base=_api_base(wire, _MANTLE_GPT), + api_key=_API_KEY, + input_cost_per_token=_INPUT_PRICE, + output_cost_per_token=_OUTPUT_PRICE, + model_info={"mode": "chat"}, + ) + + +def _probe_or_marked(request: Request) -> Reply: + if _is_probe(request): + return _health_upstream(request) + return _upstream()(request) + + +async def _answered_burst(gateway: Gateway, model: str, count: int) -> tuple[tuple[_Call, _Answer], ...]: + calls: Final = tuple(_Call(model, "chat", False, uuid.uuid4().hex) for _ in range(count)) + answers: Final = await _burst(_proxy_url(gateway), gateway.key, calls) + return tuple((call, _assert_answered(answer, call.marker)) for call, answer in zip(calls, answers, strict=True)) + + +async def test_a_custom_priced_chat_pin_bills_its_price_and_leaves_its_unpinned_sibling_on_responses( + gateway: Gateway, +) -> None: + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _priced_chat_pin(scenario, wire) + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + pinned_calls: Final = await _answered_burst(gateway, pinned, 8) + sibling_calls: Final = await _answered_burst(gateway, sibling, 8) + assert _routes(_attempts(wire)) == {call.marker: ("chat",) for call, _ in pinned_calls} | { + call.marker: ("responses",) for call, _ in sibling_calls + } + _, answer = pinned_calls[0] + rows: Final = eventually( + lambda: [ + row for row in read_rows(_SPEND_OF_MODEL_GROUP, (pinned,)) if _logs_answer(row["request_id"], answer) + ], + lambda found: len(found) == 1, + seconds=70, + ) + assert float(str(rows[0]["spend"])) == pytest.approx(5 * _INPUT_PRICE + 3 * _OUTPUT_PRICE), rows + + +async def test_a_health_probe_of_a_custom_priced_chat_pin_leaves_its_unpinned_sibling_on_responses( + gateway: Gateway, +) -> None: + with wire_server(_probe_or_marked) as wire, gateway.scenario() as scenario: + pinned: Final = _priced_chat_pin(scenario, wire) + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + async with httpx.AsyncClient(base_url=_proxy_url(gateway), timeout=60, trust_env=False) as client: + probes: Final = await asyncio.gather( + *( + client.get("/health", params={"model": pinned}, headers={"Authorization": f"Bearer {gateway.key}"}) + for _ in range(6) + ) + ) + probed: Final = tuple( + _api(request.target) + for request in wire.drain() + if request.method == "POST" and _MARKER.search(request.body.decode()) is None + ) + sibling_calls: Final = await _answered_burst(gateway, sibling, 8) + assert _routes(_attempts(wire)) == {call.marker: ("responses",) for call, _ in sibling_calls} + assert [(probe.status_code, _JSON_OBJECT.validate_json(probe.content)["healthy_count"]) for probe in probes] == [ + (200, 1) + ] * 6, [probe.text for probe in probes] + assert probed == ("chat",) * 6, probed + + +_PROBED: Final = "routing-probed-deployment" +_BACKGROUND_HEALTH_CHECKS: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"background_health_checks": True, "health_check_interval": 1} +) + + +def _failing_probes(probes: SimpleQueue[Api]) -> Callable[[Request], Reply]: + respond: Final = _upstream() + + def fail_probe(request: Request) -> Reply: + if not _is_probe(request): + return respond(request) + probes.put(_api(request.target)) + return _error(404, "The model `openai.gpt-5.6-terra` does not exist.", "invalid_request_error") + + return fail_probe + + +def _probed_deployment_config(wire: Wire, tmp_path: Path, mode: str | None, general: Mapping[str, JsonValue]) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + params: Final[dict[str, JsonValue]] = { + "model": _MANTLE_GPT, + "custom_llm_provider": "bedrock_mantle", + "api_base": wire.url + "/openai/v1", + "api_key": _API_KEY, + } + pin: Final[dict[str, JsonValue]] = {} if mode is None else {"model_info": {"mode": mode}} + config: Final = { + **base, + "model_list": [{"model_name": _PROBED, "litellm_params": params, **pin}], + "general_settings": {**object_value(base["general_settings"]), "store_model_in_db": False, **general}, + "router_settings": { + **object_value(base["router_settings"]), + "disable_cooldowns": False, + "cooldown_time": 60, + "num_retries": 0, + }, + } + path: Final = tmp_path / "deployment-mode-routing-failing-probe.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _drained(probes: SimpleQueue[Api]) -> tuple[Api, ...]: + return tuple(probes.get() for _ in range(probes.qsize())) + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +@pytest.mark.parametrize(("mode", "route"), (("chat", "chat"), (None, "responses")), ids=("chat-pin", "unpinned")) +async def test_a_failing_health_probe_leaves_its_deployment_serving_the_next_requests( + gateway: Gateway, tmp_path: Path, mode: str | None, route: Api +) -> None: + probes: Final[SimpleQueue[Api]] = SimpleQueue() + calls: Final = tuple(_Call(_PROBED, "chat", False, uuid.uuid4().hex) for _ in range(4)) + with wire_server(_failing_probes(probes)) as wire: + config: Final = _probed_deployment_config(wire, tmp_path, mode, {}) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + health: Final = owned.gateway.request("GET", "/health", params={"model": _PROBED}) + answers: Final = [await _http(owned.gateway, call) for call in calls] + served: Final = _routes(_attempts(wire, skip_probes=True)) + counts: Final = _JSON_OBJECT.validate_json(health.content) + assert (counts["healthy_count"], counts["unhealthy_count"]) == (0, 1), health.text + assert _drained(probes) == (route,) + for call, answer in zip(calls, answers, strict=True): + _assert_answered(answer, call.marker) + assert served == {call.marker: (route,) for call in calls} + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +async def test_failing_background_health_probes_leave_a_chat_pinned_deployment_serving_requests( + gateway: Gateway, tmp_path: Path +) -> None: + probes: Final[SimpleQueue[Api]] = SimpleQueue() + calls: Final = tuple(_Call(_PROBED, "chat", False, uuid.uuid4().hex) for _ in range(4)) + with wire_server(_failing_probes(probes)) as wire: + config: Final = _probed_deployment_config(wire, tmp_path, "chat", _BACKGROUND_HEALTH_CHECKS) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + await asyncio.to_thread(eventually, probes.qsize, lambda size: size >= 2, 60) + answers: Final = [await _http(owned.gateway, call) for call in calls] + served: Final = _routes(_attempts(wire, skip_probes=True)) + probed: Final = _drained(probes) + assert len(probed) >= 2 and set(probed) == {"chat"}, probed + for call, answer in zip(calls, answers, strict=True): + _assert_answered(answer, call.marker) + assert served == {call.marker: ("chat",) for call in calls} + + +@pytest.mark.parametrize( + "extra_body", + ( + {"reasoning_effort": "low", "reasoningSummary": "auto"}, + {"reasoning_effort": {"effort": "low"}, "reasoning_summary": "auto"}, + ), + ids=("string-effort", "object-effort"), +) +@pytest.mark.parametrize( + ("backend", "mode"), ((_OPENAI_GPT, None), (_MANTLE_GPT, "chat")), ids=("openai", "mantle-chat-pin") +) +async def test_a_summary_alias_beside_an_extra_body_effort_reaches_the_responses_api( + gateway: Gateway, extra_body: Mapping[str, JsonValue], backend: str, mode: str | None +) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + model: Final = _deployment(scenario, wire, backend, mode) + answer: Final = _assert_answered( + await _http(gateway, _Call(model, "chat", False, marker), {**_NO_RETRIES, "extra_body": dict(extra_body)}), + marker, + ) + (attempt,) = _attempts(wire) + assert (attempt.api, attempt.body.get("reasoning")) == ("responses", {"effort": "low", "summary": "auto"}), ( + attempt + ) + _assert_logged(answer, model, "success") + + +@pytest.mark.parametrize("placement", ("top-level", "extra-body")) +async def test_a_mantle_chat_pin_rejects_n_above_one_before_the_upstream(gateway: Gateway, placement: str) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + answer: Final = await _http( + gateway, _Call(pinned, "chat", False, marker), {**_NO_RETRIES, **_placed(placement, {"n": 2})} + ) + assert answer.status == 400, answer.text + assert "does not support n=2" in answer.text and "drop_params" in answer.text, answer.text + assert _attempts(wire) == () + _assert_logged(answer, pinned, "failure") + + +@pytest.mark.parametrize("placement", ("top-level", "extra-body")) +async def test_a_mantle_chat_pin_drops_n_above_one_under_drop_params(gateway: Gateway, placement: str) -> None: + marker: Final = uuid.uuid4().hex + extra: Final[Mapping[str, JsonValue]] = {**_NO_RETRIES, **_placed(placement, {"n": 2}), "drop_params": True} + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + answer: Final = _assert_answered(await _http(gateway, _Call(pinned, "chat", False, marker), extra), marker) + (attempt,) = _attempts(wire) + assert attempt.api == "chat" and "n" not in attempt.body, attempt + _assert_logged(answer, pinned, "success") + + +@pytest.mark.parametrize("placement", ("top-level", "extra-body")) +async def test_a_no_provider_mantle_chat_pin_sends_an_effort_object_as_its_level( + gateway: Gateway, placement: str +) -> None: + marker: Final = uuid.uuid4().hex + effort: Final[Mapping[str, JsonValue]] = {"reasoning_effort": {"effort": "low"}} + extra: Final[Mapping[str, JsonValue]] = {**_NO_RETRIES, **_placed(placement, effort)} + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat", provider_pin=False) + answer: Final = _assert_answered(await _http(gateway, _Call(pinned, "chat", False, marker), extra), marker) + (attempt,) = _attempts(wire) + assert (attempt.api, attempt.body.get("reasoning_effort")) == ("chat", "low"), attempt + _assert_logged(answer, pinned, "success") + + +@pytest.mark.parametrize( + ("raw_model_info", "own_route"), + ( + ('{"mode": ""}', "responses"), + ('{"mode": "embedding"}', "responses"), + ('{"mode": null}', "responses"), + ('{"mode": 5}', None), + ('{"mode": ["chat"]}', None), + ('{"mode": "' + "c" * 5000 + '"}', "responses"), + ('{"mode": "responses", "mode": "chat"}', "chat"), + ), + ids=("empty", "unroutable", "null", "int", "list", "5kb", "twice"), +) +async def test_a_malformed_mode_never_moves_an_unpinned_sibling( + gateway: Gateway, raw_model_info: str, own_route: Api | None +) -> None: + odd_marker: Final = uuid.uuid4().hex + sibling_markers: Final = tuple(uuid.uuid4().hex for _ in range(4)) + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + odd: Final = _raw_model_info_deployment(scenario, wire, raw_model_info) + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + if own_route is not None: + _assert_answered(await _http(gateway, _Call(odd, "chat", False, odd_marker)), odd_marker) + for marker in sibling_markers: + _assert_answered(await _http(gateway, _Call(sibling, "chat", False, marker)), marker) + own: Final[dict[str, tuple[Api, ...]]] = {} if own_route is None else {odd_marker: (own_route,)} + assert _routes(_attempts(wire)) == own | {marker: ("responses",) for marker in sibling_markers} + + +async def test_a_bridged_sibling_reports_the_upstream_401_after_one_attempt(gateway: Gateway) -> None: + marker: Final = uuid.uuid4().hex + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + _deployment(scenario, wire, _MANTLE_GPT, "chat") + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None, api_key="wrong-bearer") + answer: Final = await _http(gateway, _Call(sibling, "chat", False, marker)) + assert answer.status == 401, answer.text + assert "Invalid bearer token." in answer.text, answer.text + assert _routes(_attempts(wire)) == {marker: ("responses",)} + _assert_logged(answer, sibling, "failure") + + +async def test_adding_and_deleting_a_chat_pin_under_traffic_never_moves_the_sibling(gateway: Gateway) -> None: + with wire_server(_upstream()) as wire, gateway.scenario() as scenario: + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + before: Final = _waves(sibling, 12) + during: Final = _waves(sibling, 12) + after: Final = _waves(sibling, 12) + first: Final = asyncio.create_task(_burst(_proxy_url(gateway), gateway.key, before)) + with gateway.scenario() as pin_scenario: + await asyncio.to_thread(_deployment, pin_scenario, wire, _MANTLE_GPT, "chat") + answered_before: Final = await first + answered_during: Final = await _burst(_proxy_url(gateway), gateway.key, during) + answered_after: Final = await _burst(_proxy_url(gateway), gateway.key, after) + calls: Final = (*before, *during, *after) + answers: Final = tuple( + _assert_answered(answer, call.marker) + for call, answer in zip(calls, (*answered_before, *answered_during, *answered_after), strict=True) + ) + assert _routes(_attempts(wire)) == {call.marker: ("responses",) for call in calls} + for answer in answers: + _assert_logged(answer, sibling, "success") + + +async def test_an_upstream_outage_mid_burst_reaches_callers_and_routing_recovers(gateway: Gateway) -> None: + down: Final = threading.Event() + with wire_server(_upstream(down)) as wire, gateway.scenario() as scenario: + pinned: Final = _deployment(scenario, wire, _MANTLE_GPT, "chat") + sibling: Final = _deployment(scenario, wire, _MANTLE_GPT, None) + outage: Final = (*_waves(pinned, 10), *_waves(sibling, 10)) + recovery: Final = (*_waves(pinned, 10), *_waves(sibling, 10)) + down.set() + failed: Final = await _burst(_proxy_url(gateway), gateway.key, outage) + down.clear() + recovered: Final = tuple( + _assert_answered(answer, call.marker) + for call, answer in zip(recovery, await _burst(_proxy_url(gateway), gateway.key, recovery), strict=True) + ) + for call, answer in zip(outage, failed, strict=True): + assert answer is not None and answer.status == 503, (call, answer) + assert "The server is overloaded." in answer.text, answer.text + _assert_logged(answer, call.model, "failure") + assert _routes(_attempts(wire)) == { + call.marker: (_expected_route(call, pinned),) for call in (*outage, *recovery) + } + for call, answer in zip(recovery, recovered, strict=True): + _assert_logged(answer, call.model, "success") + + +def _yaml_config(wire: Wire, tmp_path: Path, order: tuple[str, ...], label: str) -> Path: + base: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + params: Final[dict[str, JsonValue]] = { + "model": _MANTLE_GPT, + "custom_llm_provider": "bedrock_mantle", + "api_base": wire.url + "/openai/v1", + "api_key": _API_KEY, + } + entries: Final[Mapping[str, dict[str, JsonValue]]] = { + _PINNED: {"model_name": _PINNED, "litellm_params": params, "model_info": {"mode": "chat"}}, + _SIBLING: {"model_name": _SIBLING, "litellm_params": params}, + } + config: Final = { + **base, + "model_list": [entries[name] for name in order], + "router_settings": {**object_value(base["router_settings"]), "num_retries": 0}, + } + path: Final = tmp_path / f"deployment-mode-routing-{label}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +def _holding_upstream(held: SimpleQueue[str], release: threading.Event) -> Callable[[Request], Reply]: + respond: Final = _upstream() + + def hold(request: Request) -> Reply: + if request.method == "POST": + held.put(_marker_in(request.body.decode())) + assert release.wait(timeout=60), "The burst was never released" + return respond(request) + + return hold + + +def _held_upstream_calls(pid: int, wire: Wire) -> int: + port: Final = urlsplit(wire.url).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +async def _follow_up(gateway: Gateway) -> tuple[_Call, ...]: + calls: Final = (*_waves(_PINNED, 6), *_waves(_SIBLING, 6)) + answers: Final = await _burst(_proxy_url(gateway), gateway.key, calls) + for call, answer in zip(calls, answers, strict=True): + _assert_logged(_assert_answered(answer, call.marker), call.model, "success") + return calls + + +@pytest.mark.timeout(5 * graceful_stop_seconds() + 120) +async def test_a_worker_kill_and_a_restart_in_either_order_keep_each_deployment_on_its_mode( + gateway: Gateway, tmp_path: Path +) -> None: + held: Final[SimpleQueue[str]] = SimpleQueue() + release: Final = threading.Event() + burst_calls: Final = (*_waves(_PINNED, 10), *_waves(_SIBLING, 10)) + with wire_server(_holding_upstream(held, release)) as wire: + sibling_first: Final = _yaml_config(wire, tmp_path, (_SIBLING, _PINNED), "sibling-first") + with owned_proxy_process(gateway, tmp_path, {}, config=sibling_first, workers=2) as owned: + workers: Final = eventually( + lambda: tuple(int(found.group(1)) for found in _STARTED_WORKER.finditer(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + burst: Final = asyncio.create_task( + _burst(_proxy_url(owned.gateway), owned.gateway.key, burst_calls, tolerate_drops=True) + ) + await asyncio.to_thread(eventually, held.qsize, lambda size: size == len(burst_calls), 60) + held_by_worker: Final = MappingProxyType({pid: _held_upstream_calls(pid, wire) for pid in workers}) + assert sum(held_by_worker.values()) == len(burst_calls), held_by_worker + victim: Final = max(workers, key=held_by_worker.__getitem__) + with owned.replacing_worker(victim) as kill: + kill.freeze() + await asyncio.to_thread(kill) + release.set() + served: Final = await burst + survivors: Final = tuple( + _assert_answered(answer, call.marker) + for call, answer in zip(burst_calls, served, strict=True) + if answer is not None + ) + assert len(survivors) == len(burst_calls) - held_by_worker[victim], (held_by_worker, served) + after_kill: Final = await _follow_up(owned.gateway) + pinned_first: Final = _yaml_config(wire, tmp_path, (_PINNED, _SIBLING), "pinned-first") + with owned_proxy_process(gateway, tmp_path, {}, config=pinned_first, workers=2) as restarted: + after_restart: Final = await _follow_up(restarted.gateway) + assert _routes(_attempts(wire)) == { + call.marker: (_expected_route(call, _PINNED),) for call in (*burst_calls, *after_kill, *after_restart) + } diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 83c59480d37..9aa494c1aab 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -82,6 +82,115 @@ def test_completion_over_responses_maps_anthropic_search_tool_to_openai_tool_sea assert request_body["tools"][0] == {"type": "tool_search", "execution": "server"} +_MANTLE_RESPONSES_URL: Final = "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses" + + +@pytest.mark.parametrize( + "extra_body", + [ + {"reasoning_effort": "low", "reasoningSummary": "auto"}, + {"reasoning_effort": {"effort": "low"}, "reasoning_summary": "auto"}, + ], + ids=["string-effort", "dict-effort"], +) +@pytest.mark.parametrize( + ("model", "url", "model_info"), + [ + ("openai/gpt-5.4", "https://api.openai.com/v1/responses", None), + ("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, {"mode": "chat"}), + ], + ids=["openai", "mantle-chat-pin"], +) +def test_a_summary_alias_beside_an_extra_body_effort_reaches_the_responses_api( + monkeypatch: pytest.MonkeyPatch, + extra_body: Mapping[str, object], + model: str, + url: str, + model_info: Mapping[str, object] | None, +) -> None: + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + response_body: Final = { + "id": "resp_nested_effort_summary", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": model.rsplit("/", 1)[-1], + "output": [ + { + "type": "message", + "id": "msg_nested_effort_summary", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + + with respx.mock(assert_all_called=True) as router: + upstream: Final = router.post(url).mock(return_value=httpx.Response(200, json=response_body)) + litellm.completion( + model=model, + messages=[{"role": "user", "content": "ping"}], + extra_body=dict(extra_body), + model_info=model_info, + api_key="test-key", + aws_region_name="us-east-1", + ) + + assert json.loads(upstream.calls.last.request.content)["reasoning"] == {"effort": "low", "summary": "auto"} + + +@pytest.mark.parametrize( + ("model", "url", "expected_summary"), + [ + ("bedrock_mantle/openai.gpt-5.6-terra", _MANTLE_RESPONSES_URL, "auto"), + ("openai/responses/gpt-5.5", "https://api.openai.com/v1/responses", "detailed"), + ], +) +@pytest.mark.parametrize("endpoint", ["chat_completions", "responses"]) +def test_auto_summary_sends_the_level_the_target_accepts( + monkeypatch: pytest.MonkeyPatch, model: str, url: str, expected_summary: str, endpoint: str +) -> None: + monkeypatch.setattr(litellm, "reasoning_auto_summary", True) + monkeypatch.delenv("AWS_BEARER_TOKEN_BEDROCK", raising=False) + response_body: Final = { + "id": "resp_auto_summary", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": model.rsplit("/", 1)[-1], + "output": [ + { + "type": "message", + "id": "msg_auto_summary", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + + with respx.mock(assert_all_called=True) as router: + upstream: Final = router.post(url).mock(return_value=httpx.Response(200, json=response_body)) + if endpoint == "chat_completions": + litellm.completion( + model=model, + messages=[{"role": "user", "content": "ping"}], + reasoning_effort="low", + api_key="test-key", + aws_region_name="us-east-1", + ) + else: + litellm.responses( + model=model, input="ping", reasoning_effort="low", api_key="test-key", aws_region_name="us-east-1" + ) + + request_body: Final = json.loads(upstream.calls[0].request.content) + assert request_body["reasoning"] == {"effort": "low", "summary": expected_summary} + + def test_convert_chat_completion_messages_to_responses_api_image_input(): from litellm.completion_extras.litellm_responses_transformation.transformation import ( LiteLLMResponsesTransformationHandler, diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py index 4b96d2e7991..389398dcbf4 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_handler_reasoning_effort_normalization.py @@ -49,9 +49,12 @@ def _completion_kwargs_for_thinking( *, tools: list[dict[str, object]] | None = None, api_base: str | None = None, + model_info: dict[str, str] | None = None, ) -> Mapping[str, object]: extra_kwargs: Final = { - key: value for key, value in (("custom_llm_provider", provider), ("api_base", api_base)) if value is not None + key: value + for key, value in (("custom_llm_provider", provider), ("api_base", api_base), ("model_info", model_info)) + if value is not None } completion_kwargs, _ = LiteLLMMessagesToCompletionTransformationHandler._prepare_completion_kwargs( max_tokens=1024, @@ -80,10 +83,11 @@ def _reasoning_effort_sent_for_thinking( *, tools: list[dict[str, object]] | None = None, api_base: str | None = None, + model_info: dict[str, str] | None = None, ) -> object: - return _completion_kwargs_for_thinking(model, provider, thinking, tools=tools, api_base=api_base).get( - "reasoning_effort" - ) + return _completion_kwargs_for_thinking( + model, provider, thinking, tools=tools, api_base=api_base, model_info=model_info + ).get("reasoning_effort") SUMMARIZED_THINKING = {"type": "enabled", "budget_tokens": 4096, "summary": "auto"} @@ -183,6 +187,22 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: assert sent == "high" + def test_a_chat_pinned_deployment_gets_the_plain_tier_on_a_responses_row( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A ``mode: chat`` pin sends that deployment to chat completions even though its shared row + says ``responses``, so the adapter has to read the pin too or chat gets the Responses-only dict.""" + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + model: Final = "bedrock_mantle/openai.gpt-5.4" + + unpinned = _reasoning_effort_sent_for_thinking(model, "bedrock_mantle", PLAIN_THINKING) + pinned = _reasoning_effort_sent_for_thinking( + model, "bedrock_mantle", PLAIN_THINKING, model_info={"id": "pinned-id", "mode": "chat"} + ) + + assert isinstance(unpinned, dict) + assert pinned == "high" + @pytest.mark.parametrize( "model, provider", [ @@ -205,6 +225,23 @@ class TestTheSummaryWrappingOnlyRidesTheResponsesBridge: assert sent == {"effort": "high", "summary": "detailed"} + @pytest.mark.parametrize( + "model, provider, summary", + [ + ("bedrock_mantle/openai.gpt-5.4", "bedrock_mantle", "auto"), + ("azure/responses/gpt-5-mini", "azure", "detailed"), + ], + ) + def test_auto_summary_asks_each_bridged_target_for_a_level_it_accepts( + self, local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, model: str, provider: str, summary: str + ) -> None: + """Mantle's OpenAI path accepts only ``auto``, so the opt-in summary must not send it ``detailed``.""" + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + + sent = _reasoning_effort_sent_for_thinking(model, provider, PLAIN_THINKING) + + assert sent == {"effort": "high", "summary": summary} + @pytest.mark.parametrize( "api_base, expected", [ diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 0b68f54c504..e44cdf2080b 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -1,10 +1,12 @@ import asyncio +import copy import json import os import uuid from typing import Any, Dict, Final, List import httpx +import respx import pytest from fastapi.testclient import TestClient from pydantic import ValidationError @@ -25,6 +27,7 @@ from litellm.types.utils import ( StandardLoggingPayloadErrorInformation, StreamingChoices, ) +from litellm.utils import _invalidate_model_cost_lowercase_map import unittest.mock @@ -1075,6 +1078,44 @@ async def test_messages_strips_provider_prefix_exactly_once(requested_model, exp assert captured["url"] == expected_url +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model_info, expected_url", + [ + (None, "https://api.perplexity.ai/v1/responses"), + ({"id": "pinned-id", "mode": "chat"}, "https://api.perplexity.ai/chat/completions"), + ], +) +async def test_messages_honors_a_chat_pin_on_a_prefixed_responses_deployment( + monkeypatch: pytest.MonkeyPatch, model_info: dict[str, str] | None, expected_url: str +) -> None: + """A deployment id whose stripped form is a chat row is kept on the Responses API, unless the + deployment pins ``mode: chat``, which has to decide the route before the prefix is stripped.""" + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + litellm.model_cost["perplexity/perplexity/pin-test-model"] = { + "litellm_provider": "perplexity", + "mode": "responses", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + } + litellm.model_cost["perplexity/pin-test-model"] = {"litellm_provider": "perplexity", "mode": "chat"} + _invalidate_model_cost_lowercase_map() + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=True) as respx_mock: + upstream: Final = respx_mock.route(host="api.perplexity.ai").mock(side_effect=httpx.ConnectError("cut at the wire")) + with pytest.raises(litellm.exceptions.InternalServerError): + await litellm.anthropic.messages.acreate( + max_tokens=100, + messages=[{"role": "user", "content": "ping"}], + model="perplexity/perplexity/pin-test-model", + api_key="test-api-key", + model_info=model_info, + ) + + assert {str(call.request.url) for call in upstream.calls} == {expected_url} + + @pytest.mark.asyncio async def test_native_messages_strips_replayed_provider_specific_fields_from_wire(): captured = {} diff --git a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py index 453429df084..b6f831d4bef 100644 --- a/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py +++ b/tests/unit/llms/anthropic/pass_through/responses_adapters/test_responses_adapters_handler.py @@ -3,7 +3,7 @@ import json import os import sys from collections.abc import Mapping -from typing import Final +from typing import Final, cast from unittest.mock import AsyncMock, patch import pytest @@ -110,6 +110,27 @@ def test_build_responses_kwargs_skips_include_for_a_responses_provider_that_reje assert "reasoning" in responses_kwargs +@pytest.mark.parametrize( + "model, provider, summary", + [ + ("bedrock_mantle/openai.gpt-5.6-terra", "bedrock_mantle", "auto"), + ("openai/gpt-5.6-luna", "openai", "detailed"), + ], +) +def test_build_responses_kwargs_auto_summary_asks_each_target_for_a_level_it_accepts( + monkeypatch: pytest.MonkeyPatch, model: str, provider: str, summary: str +) -> None: + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + responses_kwargs: Final = _build_responses_kwargs( + max_tokens=1024, + messages=MESSAGES, + model=model, + thinking={"type": "enabled", "budget_tokens": 4096}, + extra_kwargs={"custom_llm_provider": provider}, + ) + assert cast(Mapping[str, object], responses_kwargs["reasoning"])["summary"] == summary + + def test_build_responses_kwargs_keeps_the_deployment_include_next_to_encrypted_reasoning(): responses_kwargs = _build_responses_kwargs( max_tokens=1024, diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index 951911ac066..a2289167ad4 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -625,6 +625,38 @@ class TestBedrockMantleReasoningSummary: ) assert params["reasoning"] == {"effort": "medium", "summary": "auto"} + def test_auto_summary_on_still_drops_a_client_detailed_summary(self, monkeypatch): + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + cfg = BedrockMantleResponsesAPIConfig() + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": "detailed"}}, + model="openai.gpt-5.6-sol", + drop_params=True, + ) + assert params["reasoning"] == {"effort": "medium"} + + def test_auto_summary_on_still_rejects_a_client_detailed_summary_without_drop_params(self, monkeypatch): + monkeypatch.setenv("LITELLM_REASONING_AUTO_SUMMARY", "true") + cfg = BedrockMantleResponsesAPIConfig() + with pytest.raises(litellm.UnsupportedParamsError): + cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": "detailed"}}, + model="openai.gpt-5.6-sol", + drop_params=False, + ) + + @pytest.mark.parametrize("use_openai_path, expected", [(True, "auto"), (False, "detailed")]) + def test_auto_reasoning_summary_is_a_level_the_path_accepts(self, use_openai_path, expected): + cfg = BedrockMantleResponsesAPIConfig(use_openai_path=use_openai_path) + summary = cfg.auto_reasoning_summary() + assert summary == expected + params = cfg.map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium", "summary": summary}}, + model="openai.gpt-5.6-sol" if use_openai_path else "openai.gpt-oss-120b", + drop_params=False, + ) + assert params["reasoning"] == {"effort": "medium", "summary": summary} + def test_reasoning_summary_kept_on_standard_path(self): cfg = BedrockMantleResponsesAPIConfig(use_openai_path=False) params = cfg.map_openai_params( diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py index a148ef1f57d..ee41af2110d 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_transformation.py @@ -21,6 +21,7 @@ import litellm from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.bedrock_mantle.chat.transformation import BedrockMantleChatConfig from litellm.llms.bedrock.base_aws_llm import sign_request_off_loop_if_aws +from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.utils import LlmProviders from tests.unit.llms.bedrock.event_loop_probe import EventLoopProbe @@ -42,6 +43,63 @@ def local_cost_map(monkeypatch): litellm.get_model_info.cache_clear() +_RESPONSES_API_REPLY: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1733529600, + "status": "completed", + "model": "openai.gpt-5.6-terra", + "output": [ + { + "type": "message", + "id": "msg_test", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, +} + + +_CHAT_COMPLETIONS_REPLY: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1733529600, + "model": "openai.gpt-5.6-terra", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _use_mantle_bearer(monkeypatch: pytest.MonkeyPatch) -> None: + for var in ("AWS_BEARER_TOKEN_BEDROCK", "BEDROCK_MANTLE_API_BASE", "BEDROCK_MANTLE_REGION"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("BEDROCK_MANTLE_API_KEY", "test-key") + + +def _reply_by_route(request: httpx.Request) -> httpx.Response: + reply: Final = _RESPONSES_API_REPLY if request.url.path.endswith("/responses") else _CHAT_COMPLETIONS_REPLY + return httpx.Response(status_code=200, json=reply, request=request) + + +def _recording_client(handler: Mock) -> HTTPHandler: + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def _mantle_chat_request(drop_params: bool, **params: object) -> dict[str, object]: + optional_params: Final = litellm.get_optional_params( + model="openai.gpt-5.6-terra", custom_llm_provider="bedrock_mantle", drop_params=drop_params, **params + ) + return BedrockMantleChatConfig().transform_request( + model="openai.gpt-5.6-terra", + messages=[{"role": "user", "content": "hello"}], + optional_params=optional_params, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + class TestBedrockMantleProviderRegistration: def test_provider_enum_exists(self): assert LlmProviders.BEDROCK_MANTLE == "bedrock_mantle" @@ -273,6 +331,110 @@ class TestBedrockMantleConfig: assert optional_params["max_completion_tokens"] == 50 assert "max_tokens" not in optional_params + @pytest.mark.parametrize( + "reasoning_effort, expected", + [({"effort": "low", "summary": "auto"}, "low"), ("medium", "medium"), ({"summary": "auto"}, None)], + ) + def test_chat_completions_gets_reasoning_effort_as_a_level(self, reasoning_effort, expected) -> None: + """Mantle chat answers a Responses-shaped ``{"effort", "summary"}`` with ``invalid_type``.""" + request = _mantle_chat_request(drop_params=True, reasoning_effort=reasoning_effort) + assert request.get("reasoning_effort") == expected + + @pytest.mark.parametrize("params", [{"n": 2}, {"reasoning_effort": {"effort": "low", "summary": "auto"}}]) + def test_chat_completions_answers_400_for_what_mantle_chat_cannot_serve_without_drop_params(self, params) -> None: + with pytest.raises(litellm.UnsupportedParamsError): + _mantle_chat_request(drop_params=False, **params) + + @pytest.mark.parametrize("n, drop_params, expected", [(1, False, 1), (2, True, None)]) + def test_chat_completions_sends_n_only_where_mantle_chat_serves_it(self, n, drop_params, expected) -> None: + assert _mantle_chat_request(drop_params=drop_params, n=n).get("n") == expected + + @pytest.mark.parametrize( + "params, model_info", + [ + ({"n": 2}, None), + ({"reasoning_effort": {"effort": "low", "summary": "auto"}}, {"mode": "chat"}), + ({"web_search_options": {"search_context_size": "low"}}, {"mode": "chat"}), + ], + ) + def test_what_mantle_chat_cannot_serve_reaches_the_responses_api( + self, monkeypatch, local_cost_map, params, model_info + ) -> None: + _use_mantle_bearer(monkeypatch) + handler: Final = Mock(side_effect=_reply_by_route) + + response: Final = litellm.completion( + model="bedrock_mantle/openai.gpt-5.6-terra", + messages=[{"role": "user", "content": "hello"}], + aws_region_name="us-east-1", + model_info=model_info, + client=_recording_client(handler), + **params, + ) + + sent: Final = tuple(call.args[0] for call in handler.call_args_list) + assert response.choices[0].message.content == "ok" + assert [request.url.path.rsplit("/", 1)[-1] for request in sent] == ["responses"] + assert json.loads(sent[0].content).get("reasoning", {}).get("summary") == params.get( + "reasoning_effort", {} + ).get("summary") + + @pytest.mark.parametrize( + "extra_body, drop_params, expected", + [ + ({"reasoning_effort": {"effort": "low"}}, False, {"reasoning_effort": "low"}), + ({"n": 2}, True, {}), + ({"reasoning_effort": {}}, False, {}), + ({"reasoning_effort": {"effort": None}}, False, {}), + ({"n": 1, "reasoning_effort": "high"}, False, {"n": 1, "reasoning_effort": "high"}), + ], + ids=[ + "dict-effort", + "n-above-one-dropped", + "empty-effort-asks-for-the-default", + "null-effort-asks-for-the-default", + "servable", + ], + ) + def test_a_chat_pin_sends_extra_body_through_the_same_chat_checks( + self, monkeypatch, local_cost_map, extra_body, drop_params, expected + ) -> None: + _use_mantle_bearer(monkeypatch) + handler: Final = Mock(side_effect=_reply_by_route) + + litellm.completion( + model="bedrock_mantle/openai.gpt-5.6-terra", + messages=[{"role": "user", "content": "hello"}], + aws_region_name="us-east-1", + model_info={"mode": "chat"}, + client=_recording_client(handler), + extra_body=extra_body, + drop_params=drop_params, + ) + + (sent,) = (call.args[0] for call in handler.call_args_list) + body: Final = json.loads(sent.content) + assert sent.url.path.endswith("/chat/completions") + assert {key: body[key] for key in ("n", "reasoning_effort") if key in body} == expected + + def test_a_chat_pin_answers_400_for_n_above_one_in_extra_body_without_drop_params( + self, monkeypatch, local_cost_map + ) -> None: + _use_mantle_bearer(monkeypatch) + handler: Final = Mock(side_effect=_reply_by_route) + + with pytest.raises(litellm.BadRequestError, match="does not support n=2"): + litellm.completion( + model="bedrock_mantle/openai.gpt-5.6-terra", + messages=[{"role": "user", "content": "hello"}], + aws_region_name="us-east-1", + model_info={"mode": "chat"}, + client=_recording_client(handler), + extra_body={"n": 2}, + ) + + handler.assert_not_called() + def test_non_gpt_reasoning_models_keep_the_openai_compatible_max_tokens(self): optional_params = litellm.get_optional_params( model="openai.gpt-oss-120b", diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index cad919b09e3..24117f45106 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1,4 +1,5 @@ import asyncio +import copy import json import logging from typing import Final @@ -607,6 +608,101 @@ async def test_run_model_health_check_honors_an_explicit_chat_mode_on_mantle_cla assert fake_ahealth_check.call_args.kwargs["mode"] == "chat" +_MANTLE_GPT_CHAT_REPLY: Final = { + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1, + "model": "openai.gpt-5.6-terra", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + +_MANTLE_GPT_RESPONSES_REPLY: Final = { + "id": "resp_health", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "openai.gpt-5.6-terra", + "output": [ + { + "type": "message", + "id": "msg_health", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model_info", "probed_api"), + (({"mode": "chat"}, "chat"), ({}, "responses")), + ids=("chat-pin", "unpinned"), +) +async def test_run_model_health_check_probes_the_api_a_mantle_gpt_deployment_sends_chat_to( + monkeypatch, model_info, probed_api +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post( + url__regex=r"^https://bedrock-mantle\.us-east-2\.api\.aws/.*/chat/completions$" + ).respond(json=_MANTLE_GPT_CHAT_REPLY) + responses_route: Final = respx_mock.post( + url__regex=r"^https://bedrock-mantle\.us-east-2\.api\.aws/.*/responses$" + ).respond(json=_MANTLE_GPT_RESPONSES_REPLY) + result: Final = await hc_module._run_model_health_check( + { + "litellm_params": { + "model": "bedrock_mantle/openai.gpt-5.6-terra", + "api_key": "test-bearer", + "aws_region_name": "us-east-2", + }, + "model_info": model_info, + } + ) + + assert "error" not in result, result + assert {"chat": chat_route.call_count, "responses": responses_route.call_count} == { + "chat": int(probed_api == "chat"), + "responses": int(probed_api == "responses"), + } + + +@pytest.mark.asyncio +async def test_a_health_probe_of_a_custom_priced_chat_pin_leaves_plain_calls_on_responses(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + litellm.in_memory_llm_clients_cache.flush_cache() + params: Final = { + "model": "bedrock_mantle/openai.gpt-5.6-terra", + "api_key": "test-bearer", + "aws_region_name": "us-east-2", + } + + with respx.mock(assert_all_called=False) as respx_mock: + chat_route: Final = respx_mock.post( + url__regex=r"^https://bedrock-mantle\.us-east-2\.api\.aws/.*/chat/completions$" + ).respond(json=_MANTLE_GPT_CHAT_REPLY) + responses_route: Final = respx_mock.post( + url__regex=r"^https://bedrock-mantle\.us-east-2\.api\.aws/.*/responses$" + ).respond(json=_MANTLE_GPT_RESPONSES_REPLY) + result: Final = await hc_module._run_model_health_check( + { + "litellm_params": {**params, "input_cost_per_token": 2e-05, "output_cost_per_token": 8e-05}, + "model_info": {"mode": "chat"}, + } + ) + await litellm.acompletion(messages=[{"role": "user", "content": "Say ok"}], **params) + + assert "error" not in result, result + assert (chat_route.call_count, responses_route.call_count) == (1, 1) + + def test_autodetected_embedding_skips_reasoning_effort(): """reasoning_effort must not leak into an embedding probe whose mode is auto-detected. @@ -1313,3 +1409,36 @@ async def test_background_health_check_timeout_does_not_cancel_a_sibling( assert fast_upstream.called assert healthy_model_ids == {"fast"} assert unhealthy_model_ids == {"slow"} + + +@pytest.mark.asyncio +async def test_a_failing_health_probe_leaves_the_deployment_serving_requests(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + litellm.in_memory_llm_clients_cache.flush_cache() + deployment: Final = { + "model_name": "probe-group", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + "api_base": "https://probe.example/v1", + }, + "model_info": {"id": "probe-deployment", "mode": "chat"}, + } + router: Final = litellm.Router(model_list=[deployment]) + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.post("https://probe.example/v1/chat/completions").mock( + side_effect=( + httpx.Response(404, json={"error": {"message": "model not found", "type": "invalid_request_error"}}), + httpx.Response(200, json={**_MANTLE_GPT_CHAT_REPLY, "model": "gpt-4o-mini"}), + ) + ) + probe: Final = await hc_module._run_model_health_check(copy.deepcopy(deployment)) + reply: Final = await router.acompletion(model="probe-group", messages=[{"role": "user", "content": "Say ok"}]) + + assert "error" in probe + assert reply.choices[0].message.content == "ok" diff --git a/tests/unit/responses/test_responses_api_bridge_flag.py b/tests/unit/responses/test_responses_api_bridge_flag.py index 699144a31bb..63204114501 100644 --- a/tests/unit/responses/test_responses_api_bridge_flag.py +++ b/tests/unit/responses/test_responses_api_bridge_flag.py @@ -6,18 +6,28 @@ Includes file_search emulation: the flag must be forwarded on inner aresponses calls so routed requests do not hit a custom api_base /v1/responses endpoint. """ +import copy import json +from collections.abc import Mapping from importlib import import_module -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx import openai import pytest import respx +from pydantic import JsonValue import litellm +from litellm import Router from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.responses.bridge_check import ( + backend_serves_chat_completions, + bridge_reasoning_effort, + deployment_routing_mode, + responses_api_bridge_check, +) from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -48,12 +58,8 @@ class TestUseResponsesApiBridgeFlag: @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - def test_bridge_used_when_use_chat_completions_api_true( - self, mock_get_config, mock_bridge_handler - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + def test_bridge_used_when_use_chat_completions_api_true(self, mock_get_config, mock_bridge_handler): """When use_chat_completions_api=True, the bridge handler should be called.""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_bridge_handler.return_value = MagicMock() @@ -70,9 +76,7 @@ class TestUseResponsesApiBridgeFlag: @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") def test_provider_affinity_header_is_forwarded_through_bridge(self, mock_get_config, mock_bridge_handler): mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_bridge_handler.return_value = MagicMock() @@ -94,12 +98,8 @@ class TestUseResponsesApiBridgeFlag: @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - def test_bridge_used_when_model_uses_chat_completions_prefix( - self, mock_get_config, mock_bridge_handler - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + def test_bridge_used_when_model_uses_chat_completions_prefix(self, mock_get_config, mock_bridge_handler): """`openai/chat_completions/` normalizes to `openai/` and uses the bridge.""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_bridge_handler.return_value = MagicMock() @@ -115,12 +115,8 @@ class TestUseResponsesApiBridgeFlag: assert mock_bridge_handler.call_args.kwargs["model"].endswith("my-custom-model") @patch.object(import_module("litellm.responses.main").base_llm_http_handler, "response_api_handler") - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - def test_native_forwarding_when_flag_absent( - self, mock_get_config, mock_native_handler - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + def test_native_forwarding_when_flag_absent(self, mock_get_config, mock_native_handler): """When use_chat_completions_api is not set, openai/ models should use native responses API forwarding (existing behavior).""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() @@ -137,9 +133,7 @@ class TestUseResponsesApiBridgeFlag: @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") def test_flag_does_not_leak_into_kwargs(self, mock_get_config, mock_bridge_handler): """use_chat_completions_api should be popped and not passed to the bridge handler.""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() @@ -159,12 +153,8 @@ class TestUseResponsesApiBridgeFlag: @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - def test_bridge_used_when_provider_config_none( - self, mock_get_config, mock_bridge_handler - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + def test_bridge_used_when_provider_config_none(self, mock_get_config, mock_bridge_handler): """When the provider has no native responses API config (returns None), the bridge should be used regardless of the flag (existing behavior).""" mock_get_config.return_value = None @@ -179,12 +169,8 @@ class TestUseResponsesApiBridgeFlag: mock_bridge_handler.assert_called_once() @patch("litellm.acompletion") - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - async def test_allowed_openai_params_forwarded_through_bridge( - self, mock_get_config, mock_acompletion - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + async def test_allowed_openai_params_forwarded_through_bridge(self, mock_get_config, mock_acompletion): """allowed_openai_params is a named param of responses(), so it must be explicitly forwarded to the bridge; otherwise litellm.acompletion raises UnsupportedParamsError for params the caller explicitly allowed.""" @@ -212,9 +198,7 @@ class TestUseResponsesApiBridgeFlag: ) mock_acompletion.assert_called_once() - assert mock_acompletion.call_args.kwargs.get("allowed_openai_params") == [ - "reasoning_effort" - ] + assert mock_acompletion.call_args.kwargs.get("allowed_openai_params") == ["reasoning_effort"] @pytest.mark.parametrize( ("model", "upstream_url", "use_chat_completions_api", "allowed_openai_params", "expected_chat_template_kwargs"), @@ -287,9 +271,7 @@ class TestUseResponsesApiBridgeFlag: assert request_body["messages"] == [{"role": "user", "content": "Hello"}] assert response.output[0].content[0].text == "Answer" - def test_bridge_drops_client_metadata_even_when_allowed_openai_params_names_it( - self, respx_mock: respx.MockRouter - ): + def test_bridge_drops_client_metadata_even_when_allowed_openai_params_names_it(self, respx_mock: respx.MockRouter): upstream: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( return_value=httpx.Response( status_code=200, @@ -441,12 +423,8 @@ class TestUseResponsesApiBridgeFlag: assert "chat_template_kwargs" not in json.loads(request.read()) @patch.object(import_module("litellm.responses.file_search.emulated_handler"), "_call_aresponses") - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - async def test_bridge_flag_forwarded_to_file_search_emulation( - self, mock_get_config, mock_call_aresponses - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + async def test_bridge_flag_forwarded_to_file_search_emulation(self, mock_get_config, mock_call_aresponses): """When use_chat_completions_api=True and file_search tool is present, the flag should be forwarded to the inner aresponses call in the file_search emulation path.""" @@ -458,12 +436,8 @@ class TestUseResponsesApiBridgeFlag: id="resp_123", model="openai/my-custom-model", created_at=1234567890, - output=[ - {"type": "message", "content": [{"type": "text", "text": "Answer"}]} - ], - usage=ResponseAPIUsage( - input_tokens=10, output_tokens=5, total_tokens=15 - ), + output=[{"type": "message", "content": [{"type": "text", "text": "Answer"}]}], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), ) mock_call_aresponses.return_value = mock_response @@ -478,17 +452,15 @@ class TestUseResponsesApiBridgeFlag: # Verify _call_aresponses was called with use_chat_completions_api=True mock_call_aresponses.assert_called_once() call_kwargs = mock_call_aresponses.call_args.kwargs - assert ( - call_kwargs.get("use_chat_completions_api") is True - ), "use_chat_completions_api should be forwarded to inner aresponses call" + assert call_kwargs.get("use_chat_completions_api") is True, ( + "use_chat_completions_api should be forwarded to inner aresponses call" + ) @patch.object( import_module("litellm.responses.main").litellm_completion_transformation_handler, "response_api_handler" ) @patch("litellm.vector_stores.main.asearch") - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") async def test_bridge_flag_prevents_native_responses_endpoint_call( self, mock_get_config, mock_asearch, mock_bridge_handler ): @@ -512,9 +484,7 @@ class TestUseResponsesApiBridgeFlag: "arguments": '{"queries": ["test query"]}', } ], - usage=ResponseAPIUsage( - input_tokens=10, output_tokens=5, total_tokens=15 - ), + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), ) second_response = ResponsesAPIResponse( id="resp_second", @@ -526,9 +496,7 @@ class TestUseResponsesApiBridgeFlag: "content": [{"type": "text", "text": "Final answer"}], } ], - usage=ResponseAPIUsage( - input_tokens=20, output_tokens=10, total_tokens=30 - ), + usage=ResponseAPIUsage(input_tokens=20, output_tokens=10, total_tokens=30), ) mock_bridge_handler.side_effect = [first_response, second_response] @@ -547,8 +515,7 @@ class TestUseResponsesApiBridgeFlag: ) assert mock_bridge_handler.call_count == 2, ( - "Bridge handler should be called twice: initial function-tool call " - "and follow-up with tool results" + "Bridge handler should be called twice: initial function-tool call and follow-up with tool results" ) for call in mock_bridge_handler.call_args_list: all_kwargs = call.kwargs if call.kwargs else {} @@ -558,12 +525,8 @@ class TestUseResponsesApiBridgeFlag: @patch.object(import_module("litellm.responses.main").base_llm_http_handler, "response_api_handler") @patch("litellm.vector_stores.main.asearch") - @patch.object( - import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config" - ) - async def test_without_bridge_flag_uses_native_endpoint( - self, mock_get_config, mock_asearch, mock_native_handler - ): + @patch.object(import_module("litellm.responses.main").ProviderConfigManager, "get_provider_responses_api_config") + async def test_without_bridge_flag_uses_native_endpoint(self, mock_get_config, mock_asearch, mock_native_handler): """Without the bridge flag, openai/ with native config uses the native handler.""" mock_get_config.return_value = litellm.OpenAIResponsesAPIConfig() mock_asearch.return_value = [] @@ -577,9 +540,7 @@ class TestUseResponsesApiBridgeFlag: "content": [{"type": "text", "text": "Native response"}], } ], - usage=ResponseAPIUsage( - input_tokens=10, output_tokens=5, total_tokens=15 - ), + usage=ResponseAPIUsage(input_tokens=10, output_tokens=5, total_tokens=15), ) result = await litellm.aresponses( @@ -607,3 +568,287 @@ class TestUseResponsesApiBridgeFlag: assert exc_info.value.param == "stream_chunk_size" send.assert_not_called() + + +# Deployment-pinned routing mode (deployment_routing_mode / responses_api_bridge_check) +_CHAT: Final[Mapping[str, JsonValue]] = {"id": "dep-1", "mode": "chat"} +_RESPONSES: Final[Mapping[str, JsonValue]] = {"id": "dep-1", "mode": "responses"} + + +def _pin_shared_mode(monkeypatch: pytest.MonkeyPatch, model: str, mode: str) -> None: + cost_map: Final = cast(dict[str, dict[str, object]], litellm.model_cost) + monkeypatch.setitem(cost_map, model, {**cost_map.get(model, {}), "mode": mode}) + + +@pytest.mark.parametrize( + "deployment_model_info, custom_llm_provider, expected", + [ + (_CHAT, "bedrock_mantle", "chat"), + (_RESPONSES, "bedrock_mantle", "responses"), + (_CHAT, "openai", "chat"), + (_CHAT, "chatgpt", None), + (_CHAT, "github_copilot", None), + ({"id": "dep-1", "mode": "embedding"}, "bedrock_mantle", None), + ({"id": "dep-1"}, "bedrock_mantle", None), + (None, "bedrock_mantle", None), + ], +) +def test_deployment_routing_mode_pins_chat_or_responses_except_on_responses_only_backends( + deployment_model_info: Mapping[str, JsonValue] | None, custom_llm_provider: str, expected: str | None +) -> None: + assert deployment_routing_mode(deployment_model_info, custom_llm_provider) == expected + + +def test_a_chat_pinned_mantle_deployment_stays_on_chat_completions_while_the_shared_row_says_responses( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model: Final = "bedrock_mantle/openai.gpt-5.6-sol" + _pin_shared_mode(monkeypatch, model, "responses") + shared_info, _ = responses_api_bridge_check(model=model, custom_llm_provider="bedrock_mantle") + pinned_info, pinned_model = responses_api_bridge_check( + model=model, custom_llm_provider="bedrock_mantle", deployment_model_info=_CHAT + ) + assert shared_info["mode"] == "responses" + assert (pinned_info["mode"], pinned_model) == ("chat", model) + + +def test_a_chat_pinned_chatgpt_alias_still_bridges_to_the_responses_api(monkeypatch: pytest.MonkeyPatch) -> None: + _pin_shared_mode(monkeypatch, "chatgpt/gpt-5.4", "responses") + info, _ = responses_api_bridge_check(model="gpt-5.4", custom_llm_provider="chatgpt", deployment_model_info=_CHAT) + assert info["mode"] == "responses" + + +_FUNCTION_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {}}}, +} + + +@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"]) +def test_a_chat_pin_cannot_pull_gpt_5_4_function_tool_calls_off_the_responses_api(custom_llm_provider: str) -> None: + """Chat Completions rejects function tools while reasoning is on for gpt-5.4+, so the capability rule + must still bridge the request even when the deployment pins ``mode: chat``.""" + info, model = responses_api_bridge_check( + model="gpt-5.4", + custom_llm_provider=custom_llm_provider, + tools=[_FUNCTION_TOOL], + api_base="https://example.openai.azure.com" if custom_llm_provider == "azure" else None, + deployment_model_info=_CHAT, + ) + assert (info["mode"], model) == ("responses", "gpt-5.4") + + +@pytest.mark.parametrize("model", ["openai.gpt-5.6-terra", "us-east-1/openai.gpt-5.6-terra"]) +@pytest.mark.parametrize("reasoning_effort", [None, "low"]) +def test_a_chat_pinned_mantle_deployment_sends_function_tools_to_the_responses_api( + model: str, reasoning_effort: str | None +) -> None: + """Mantle Chat Completions rejects function tools on gpt-5.4+ unless reasoning is off, so a ``mode: chat`` + pin cannot keep a tool call there.""" + info, routed_model = responses_api_bridge_check( + model=model, + custom_llm_provider="bedrock_mantle", + tools=[_FUNCTION_TOOL], + reasoning_effort=reasoning_effort, + deployment_model_info=_CHAT, + ) + assert (info["mode"], routed_model) == ("responses", model) + + +@pytest.mark.parametrize( + "reasoning_effort, reasoning_summary", + [("low", "auto"), ({"effort": "low", "summary": "auto"}, None)], +) +def test_a_chat_pinned_mantle_deployment_sends_a_reasoning_summary_to_the_responses_api( + reasoning_effort: str | Mapping[str, JsonValue], reasoning_summary: str | None +) -> None: + info, _ = responses_api_bridge_check( + model="openai.gpt-5.6-terra", + custom_llm_provider="bedrock_mantle", + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + deployment_model_info=_CHAT, + ) + assert info["mode"] == "responses" + + +def test_a_chat_pinned_mantle_deployment_sends_web_search_to_the_responses_api() -> None: + info, model = responses_api_bridge_check( + model="openai.gpt-5.6-terra", + custom_llm_provider="bedrock_mantle", + web_search_options={"search_context_size": "medium"}, + deployment_model_info=_CHAT, + ) + assert (info["mode"], model) == ("responses", "openai.gpt-5.6-terra") + + +@pytest.mark.parametrize( + "model, reasoning_effort", + [ + ("openai.gpt-5.6-terra", "none"), + ("openai.gpt-oss-120b", "low"), + ("openai.gpt-oss-safeguard-120b", "low"), + ], +) +def test_a_chat_pinned_mantle_tool_call_stays_on_chat_completions_when_chat_accepts_it( + model: str, reasoning_effort: str +) -> None: + info, routed_model = responses_api_bridge_check( + model=model, + custom_llm_provider="bedrock_mantle", + tools=[_FUNCTION_TOOL], + reasoning_effort=reasoning_effort, + deployment_model_info=_CHAT, + ) + assert (info["mode"], routed_model) == ("chat", model) + + +def test_a_chat_pin_cannot_pull_xai_web_search_off_the_responses_api() -> None: + info, model = responses_api_bridge_check( + model="grok-4", + custom_llm_provider="xai", + web_search_options={"search_context_size": "medium"}, + deployment_model_info=_CHAT, + ) + assert (info["mode"], model) == ("responses", "grok-4") + + +@pytest.mark.parametrize( + "model, custom_llm_provider, api_base", + [ + ("responses/my-azure-deployment", "azure", "https://example.openai.azure.com"), + ("responses/gpt-5.4", "openai", None), + ], +) +def test_a_chat_pin_does_not_override_an_explicit_responses_prefix( + model: str, custom_llm_provider: str, api_base: str | None +) -> None: + info, sent_model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + deployment_model_info=_CHAT, + ) + assert (info["mode"], sent_model) == ("responses", model.removeprefix("responses/")) + + +def test_a_chat_pinned_responses_prefixed_router_deployment_reaches_the_responses_api( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + """The router registers a ``mode: chat`` row under the deployment's own ``responses/gpt-5.6`` name, + so the prefix has to win over that row too, or the request goes to Chat Completions as ``responses/gpt-5.6``.""" + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + responses_route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-5.6", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "prefixed-chat", + "litellm_params": { + "model": "openai/responses/gpt-5.6", + "custom_llm_provider": "openai", + "api_key": "sk-test", + }, + "model_info": {"id": "prefixed-chat-id", "mode": "chat"}, + } + ] + ) + + response: Final = router.completion(model="prefixed-chat", messages=[{"role": "user", "content": "Say ok"}]) + + assert json.loads(responses_route.calls[0].request.read())["model"] == "gpt-5.6" + assert response.choices[0].message.content == "ok" + + +def test_a_chat_pin_keeps_a_plain_gpt_5_4_chat_request_on_chat_completions() -> None: + """Without tools or a reasoning summary nothing forces the Responses API, so the deployment's own + mode decides and the shared ``responses`` row is left alone.""" + shared_mode = litellm.model_cost["gpt-5.4"]["mode"] + info, model = responses_api_bridge_check(model="gpt-5.4", custom_llm_provider="openai", deployment_model_info=_CHAT) + assert (info["mode"], model) == ("chat", "gpt-5.4") + assert litellm.model_cost["gpt-5.4"]["mode"] == shared_mode + + +@pytest.mark.parametrize( + "model, custom_llm_provider", + [ + ("gpt-5.1-codex", "azure"), + ("o3-pro", "openai"), + ("gpt-5.4-pro", "openai"), + ("eu/gpt-5-codex", "azure"), + ("us/gpt-5.4-pro", "azure"), + ("databricks-gpt-5-3-codex", "databricks"), + ], +) +def test_a_chat_pin_cannot_pull_a_responses_only_model_off_the_responses_api( + model: str, custom_llm_provider: str +) -> None: + """These rows list only ``/v1/responses`` in ``supported_endpoints``, or are ``responses`` rows with no + endpoint list at all, so a deployment pin to chat has nowhere to send the request and the bridge keeps it.""" + cost_map: Final = cast(dict[str, dict[str, object]], litellm.model_cost) + assert not backend_serves_chat_completions( + cost_map[model if custom_llm_provider == "openai" else f"{custom_llm_provider}/{model}"] + ) + info, routed_model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + api_base="https://example.openai.azure.com" if custom_llm_provider == "azure" else None, + deployment_model_info=_CHAT, + ) + assert (info["mode"], routed_model) == ("responses", model) + + +@pytest.mark.parametrize( + "shared_model_info, expected", + [ + ({}, True), + ({"mode": "chat"}, True), + ({"mode": "responses"}, False), + ({"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]}, True), + ({"supported_endpoints": ["/v1/responses", "/v1/batch"]}, False), + ({"supported_endpoints": "/v1/chat/completions"}, True), + ], +) +def test_backend_serves_chat_completions_reads_the_shared_row( + shared_model_info: Mapping[str, JsonValue], expected: bool +) -> None: + assert backend_serves_chat_completions(shared_model_info) is expected + + +@pytest.mark.parametrize( + "extra_body, expected", + [ + ({"reasoning_effort": {"effort": "low", "summary": "auto"}}, {"effort": "low", "summary": "auto"}), + ({"reasoning_effort": "low"}, "low"), + ({"reasoning_effort": 5}, None), + ({"reasoning_effort": ["low"]}, None), + ({}, None), + ("reasoning_effort=low", None), + (None, None), + ], + ids=["dict", "string", "int", "list", "absent", "not-a-mapping", "no-extra-body"], +) +def test_bridge_reasoning_effort_reads_only_a_string_or_mapping_effort_nested_in_extra_body( + extra_body: JsonValue, expected: JsonValue +) -> None: + assert bridge_reasoning_effort(None, extra_body) == expected diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index b353dc86d5f..b4a0837578b 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -10,18 +10,21 @@ should still use the built-in pricing. import asyncio import copy import itertools +import json import logging import os +import pathlib import queue import re import sys import threading from types import MappingProxyType -from typing import Final +from typing import Final, cast from unittest.mock import Mock, patch import httpx import pytest +import respx import litellm from litellm import Router @@ -44,6 +47,9 @@ from litellm.utils import ( ) +_COPILOT_TOKEN_EXPIRES_AT: Final = 4_102_444_800 + + def _simulate_price_data_reload(fetched_catalog): """Drive what a price data reload does to this process's litellm state. @@ -2130,6 +2136,398 @@ def test_backend_cost_map_keys_matches_what_registration_writes(model, custom_ll assert keys[0] == (model if custom_llm_provider is None else f"{custom_llm_provider}/{model}") +def test_mantle_chat_deployment_does_not_rewrite_the_shared_responses_row(): + """ + A Bedrock Mantle deployment that pins ``mode: chat`` routes its own chat + requests to Chat Completions, while the shared cost-map row keeps + ``mode: responses`` for every other deployment and plain SDK calls. + """ + from litellm.main import responses_api_bridge_check + + backend_model = "bedrock_mantle/openai.gpt-5.6-sol" + cost_map: Final = cast(dict[str, dict[str, object]], litellm.model_cost) + model_keys = { + key: copy.deepcopy(cost_map.get(key)) + for key in (backend_model, f"bedrock_mantle/{backend_model}", "mantle-chat-pinned") + } + + try: + litellm.model_cost[backend_model] = { + **(model_keys[backend_model] or {}), + "litellm_provider": "bedrock_mantle", + "mode": "responses", + } + _invalidate_model_cost_lowercase_map() + + Router( + model_list=[ + { + "model_name": "mantle-chat", + "litellm_params": { + "model": backend_model, + "custom_llm_provider": "bedrock_mantle", + "api_key": "test-bearer", + }, + "model_info": {"id": "mantle-chat-pinned", "mode": "chat"}, + } + ] + ) + + assert litellm.model_cost[backend_model]["mode"] == "responses" + + shared_info, _ = responses_api_bridge_check(model=backend_model, custom_llm_provider="bedrock_mantle") + assert shared_info["mode"] == "responses" + pinned_info, _ = responses_api_bridge_check( + model=backend_model, + custom_llm_provider="bedrock_mantle", + deployment_model_info={"id": "mantle-chat-pinned", "mode": "chat"}, + ) + assert pinned_info["mode"] == "chat" + finally: + _restore_model_cost_entries(model_keys) + + +def _register_routing_test_row(monkeypatch: pytest.MonkeyPatch, mode: str) -> None: + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + litellm.model_cost["routing-test-model"] = { + "litellm_provider": "openai", + "mode": mode, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "max_input_tokens": 1000, + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + } + _invalidate_model_cost_lowercase_map() + + +@pytest.mark.parametrize("registered_mode, pinned_mode", [("responses", "chat"), ("chat", "responses")]) +@pytest.mark.parametrize( + "model, custom_llm_provider", + [ + ("openai/routing-test-model", "openai"), + ("openai/routing-test-model", None), + ("routing-test-model", "openai"), + ("openai/responses/routing-test-model", None), + ], +) +def test_a_deployment_mode_never_rewrites_the_shared_row( + monkeypatch: pytest.MonkeyPatch, + registered_mode: str, + pinned_mode: str, + model: str, + custom_llm_provider: str | None, +) -> None: + """A ``mode`` pin routes only its own deployment. Every key registration merges into the shared + ```` row, the ``responses/``-stripped twin included, has to keep the mode the SDK and the + sibling deployments route by, whichever way the pin points.""" + from litellm.main import responses_api_bridge_check + + _register_routing_test_row(monkeypatch, registered_mode) + litellm_params: Final = {"model": model, "api_key": "sk-test"} | ( + {"custom_llm_provider": custom_llm_provider} if custom_llm_provider else {} + ) + + Router( + model_list=[ + { + "model_name": "pinned", + "litellm_params": litellm_params, + "model_info": {"id": "pinned-id", "mode": pinned_mode}, + }, + {"model_name": "sibling", "litellm_params": {"model": "openai/routing-test-model", "api_key": "sk-test"}}, + ] + ) + + assert litellm.model_cost["routing-test-model"]["mode"] == registered_mode + shared_info, _ = responses_api_bridge_check(model="routing-test-model", custom_llm_provider="openai") + assert shared_info["mode"] == registered_mode + + +def test_a_custom_priced_chat_pin_request_leaves_its_unpinned_sibling_on_responses( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + _register_routing_test_row(monkeypatch, "responses") + chat_route: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response( + 200, + json={ + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1, + "model": "routing-test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + ) + responses_route: Final = respx_mock.post("https://api.openai.com/v1/responses").mock( + return_value=httpx.Response( + 200, + json={ + "id": "resp_1", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "routing-test-model", + "output": [ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + ) + router: Final = Router( + model_list=[ + { + "model_name": "priced-chat-pin", + "litellm_params": { + "model": "openai/routing-test-model", + "api_key": "sk-test", + "input_cost_per_token": 3e-06, + "output_cost_per_token": 4e-06, + }, + "model_info": {"id": "priced-chat-pin-id", "mode": "chat"}, + }, + {"model_name": "sibling", "litellm_params": {"model": "openai/routing-test-model", "api_key": "sk-test"}}, + ] + ) + + router.completion(model="priced-chat-pin", messages=[{"role": "user", "content": "Say ok"}]) + router.completion(model="sibling", messages=[{"role": "user", "content": "Say ok"}]) + + assert (chat_route.call_count, responses_route.call_count) == (1, 1) + + +def test_a_prefixed_deployment_override_leaves_the_shared_row_limits_alone(monkeypatch: pytest.MonkeyPatch) -> None: + """A deployment that names its provider twice, in ``custom_llm_provider`` and in the model, registers + under its own key, so its ``max_input_tokens`` override stays off the row its siblings read.""" + _register_routing_test_row(monkeypatch, "responses") + shared_before: Final = copy.deepcopy(litellm.model_cost["routing-test-model"]) + + Router( + model_list=[ + { + "model_name": "small-window", + "litellm_params": { + "model": "openai/routing-test-model", + "custom_llm_provider": "openai", + "api_key": "sk-test", + }, + "model_info": {"id": "small-window-id", "max_input_tokens": 10}, + } + ] + ) + + assert litellm.model_cost["routing-test-model"] == shared_before + + +@pytest.mark.parametrize("pins", [("chat", "responses"), ("responses", "chat")], ids=["chat first", "responses first"]) +def test_an_unmapped_model_routes_the_same_whichever_pin_registers_first( + monkeypatch: pytest.MonkeyPatch, pins: tuple[str, str] +) -> None: + """A model missing from the cost map gets its shared row from the router, so conflicting pins must not + leave a row that depends on deployment order, and a ``responses`` pin keeps opting the model in.""" + from litellm.main import responses_api_bridge_check + + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + Router( + model_list=[ + { + "model_name": f"pinned-{mode}", + "litellm_params": {"model": "openai/unmapped-routing-test-model", "api_key": "sk-test"}, + "model_info": {"id": f"pinned-{mode}", "mode": mode}, + } + for mode in pins + ] + ) + + shared_info, _ = responses_api_bridge_check(model="unmapped-routing-test-model", custom_llm_provider="openai") + assert shared_info.get("mode") == "responses" + + +_FINE_TUNE: Final = "openai/ft:gpt-4o-mini-2024-07-18:routing-test::abc123" + + +def _fine_tune_routing( + monkeypatch: pytest.MonkeyPatch, pristine_cost_map: dict[str, object], order: tuple[str | None, ...] +) -> tuple[object, ...]: + from litellm.main import responses_api_bridge_check + + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(pristine_cost_map)) + Router( + model_list=[ + { + "model_name": f"ft-{mode}", + "litellm_params": {"model": _FINE_TUNE, "api_key": "sk-test"}, + "model_info": {"id": f"ft-{mode}"} if mode is None else {"id": f"ft-{mode}", "mode": mode}, + } + for mode in order + ] + ) + return tuple( + responses_api_bridge_check( + model=_FINE_TUNE.removeprefix("openai/"), + custom_llm_provider="openai", + deployment_model_info=None if mode is None else {"mode": mode}, + )[0].get("mode") + for mode in (None, "chat", "responses") + ) + + +@pytest.mark.parametrize("order", list(itertools.permutations(("chat", "responses", None))), ids=str) +def test_a_fine_tune_routes_the_same_whichever_deployment_registers_first( + monkeypatch: pytest.MonkeyPatch, order: tuple[str | None, ...] +) -> None: + """A fine-tune missing from the cost map inherits its base model's row on its first registration, so a + ``responses`` pin must not opt the shared row in only when it happens to register before its siblings.""" + pristine_cost_map: Final = litellm.model_cost + + assert _fine_tune_routing(monkeypatch, pristine_cost_map, order) == _fine_tune_routing( + monkeypatch, pristine_cost_map, ("responses", "chat", None) + ) + + +def test_a_non_routing_mode_pin_still_reaches_the_shared_row(monkeypatch: pytest.MonkeyPatch) -> None: + """Only ``chat`` and ``responses`` pins are per-deployment routing. Other modes, such as a realtime + model pinned to ``audio_transcription``, are still read from the shared row by name.""" + _register_routing_test_row(monkeypatch, "chat") + + Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": {"model": "openai/routing-test-model", "api_key": "sk-test"}, + "model_info": {"id": "transcriber-id", "mode": "audio_transcription"}, + } + ] + ) + + assert litellm.get_model_info(model="routing-test-model", custom_llm_provider="openai")["mode"] == ( + "audio_transcription" + ) + + +def test_a_non_routing_mode_pin_leaves_a_responses_row_on_the_responses_api(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.main import responses_api_bridge_check + + _register_routing_test_row(monkeypatch, "responses") + + Router( + model_list=[ + { + "model_name": "transcriber", + "litellm_params": {"model": "openai/routing-test-model", "api_key": "sk-test"}, + "model_info": {"id": "transcriber-id", "mode": "audio_transcription"}, + } + ] + ) + + shared_info, _ = responses_api_bridge_check(model="routing-test-model", custom_llm_provider="openai") + assert shared_info["mode"] == "responses" + + +@pytest.mark.parametrize("registered_mode", ["responses", "chat"]) +@pytest.mark.parametrize("malformed_mode", [5, ["chat"], {"mode": "chat"}], ids=["int", "list", "object"]) +def test_a_malformed_mode_never_reaches_the_shared_row( + monkeypatch: pytest.MonkeyPatch, registered_mode: str, malformed_mode: object +) -> None: + from litellm.main import responses_api_bridge_check + + _register_routing_test_row(monkeypatch, registered_mode) + + Router( + model_list=[ + { + "model_name": "malformed", + "litellm_params": {"model": "openai/routing-test-model", "api_key": "sk-test"}, + "model_info": {"id": "malformed-id", "mode": malformed_mode}, + } + ] + ) + + assert litellm.model_cost["routing-test-model"]["mode"] == registered_mode + shared_info, _ = responses_api_bridge_check(model="routing-test-model", custom_llm_provider="openai") + assert shared_info["mode"] == registered_mode + + +def test_a_github_copilot_chat_pin_opts_a_dual_endpoint_model_out_of_the_responses_api( + monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path +) -> None: + """``github_copilot`` sends a model listing both endpoints to ``/v1/responses`` unless its row says ``chat``, + so a ``chat`` pin has to reach the row to keep that model on chat completions.""" + from litellm.llms.github_copilot.responses.transformation import github_copilot_supports_responses_api + + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "api-key.json").write_text(json.dumps({"token": "tid=test", "expires_at": _COPILOT_TOKEN_EXPIRES_AT})) + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + litellm.model_cost["github_copilot/routing-test-model"] = { + "litellm_provider": "github_copilot", + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"], + "max_input_tokens": 1000, + } + _invalidate_model_cost_lowercase_map() + assert github_copilot_supports_responses_api("github_copilot/routing-test-model") is True + + Router( + model_list=[ + { + "model_name": "copilot", + "litellm_params": {"model": "github_copilot/routing-test-model"}, + "model_info": {"id": "copilot-chat-pinned", "mode": "chat"}, + } + ] + ) + + assert github_copilot_supports_responses_api("github_copilot/routing-test-model") is False + + +@pytest.mark.parametrize("registered_mode, pinned_mode", [("chat", "responses"), ("responses", "chat")]) +@pytest.mark.parametrize( + "model, custom_llm_provider", [("github_copilot/routing-test-model", None), ("routing-test-model", "github_copilot")] +) +def test_a_github_copilot_responses_pin_still_opts_the_model_into_the_responses_api( + monkeypatch: pytest.MonkeyPatch, + tmp_path: pathlib.Path, + registered_mode: str, + pinned_mode: str, + model: str, + custom_llm_provider: str | None, +) -> None: + """``github_copilot`` dispatches ``/v1/responses`` natively by the shared row's mode, and a ``responses`` + pin is its documented opt-in, while a ``chat`` pin must not move a Responses-only row off it.""" + from litellm.llms.github_copilot.responses.transformation import github_copilot_supports_responses_api + + monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path)) + (tmp_path / "api-key.json").write_text(json.dumps({"token": "tid=test", "expires_at": _COPILOT_TOKEN_EXPIRES_AT})) + monkeypatch.setattr(litellm, "model_cost", copy.deepcopy(litellm.model_cost)) + litellm.model_cost["github_copilot/routing-test-model"] = { + "litellm_provider": "github_copilot", + "mode": registered_mode, + "supported_endpoints": ["/v1/responses"] if registered_mode == "responses" else ["/v1/chat/completions"], + "max_input_tokens": 1000, + } + _invalidate_model_cost_lowercase_map() + + Router( + model_list=[ + { + "model_name": "copilot", + "litellm_params": {"model": model} + | ({"custom_llm_provider": custom_llm_provider} if custom_llm_provider else {}), + "model_info": {"id": "copilot-pinned", "mode": pinned_mode}, + } + ] + ) + + assert github_copilot_supports_responses_api("github_copilot/routing-test-model") is True + + def test_a_discarded_router_stops_contributing_to_later_reloads(monkeypatch): """ `_route_user_config_request` builds a Router per request from caller-supplied diff --git a/tests/unit/test_xai_responses_auto_routing.py b/tests/unit/test_xai_responses_auto_routing.py index c31f599dd79..89b93df1891 100644 --- a/tests/unit/test_xai_responses_auto_routing.py +++ b/tests/unit/test_xai_responses_auto_routing.py @@ -4,14 +4,16 @@ Test automatic routing to xAI Responses API when tools are present import json from collections.abc import Mapping -from typing import Final +from typing import Final, cast from unittest.mock import MagicMock, patch import httpx import pytest + import litellm from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.main import responses_api_bridge_check +from litellm.types.llms.openai import OpenAIWebSearchOptions class _RecordingResponsesHandler: @@ -46,7 +48,6 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") != "responses" assert updated_model == model - def test_responses_api_bridge_check_with_empty_tools(self): """Test that with empty tools list, xAI does not route to Responses API""" model = "grok-3" @@ -106,9 +107,6 @@ class TestXAIResponsesAutoRouting: assert model_info.get("mode") == "responses" assert updated_model == "grok-3" # prefix removed - - - def test_responses_api_bridge_check_with_web_search_options(self): """Test auto-routing with web_search_options""" model = "grok-4-1-fast" @@ -119,7 +117,7 @@ class TestXAIResponsesAutoRouting: model_info, updated_model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, + web_search_options=cast(OpenAIWebSearchOptions | None, web_search_options), ) # Should auto-route with web_search_options @@ -136,7 +134,7 @@ class TestXAIResponsesAutoRouting: model_info, updated_model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, + web_search_options=cast(OpenAIWebSearchOptions | None, web_search_options), ) # Should auto-route with both present @@ -155,9 +153,7 @@ class TestXAIResponsesAutoRouting: assert updated_model == "grok-not-in-cost-map" @patch("litellm.completion_extras.responses_api_bridge.completion") - def test_completion_with_tools_routes_to_responses_api( - self, mock_responses_completion - ): + def test_completion_with_tools_routes_to_responses_api(self, mock_responses_completion): """Test that completion() with tools routes to Responses API""" # Mock the responses_api_bridge.completion to avoid actual API calls mock_responses_completion.return_value = MagicMock()