diff --git a/litellm/constants.py b/litellm/constants.py index 7f2d98b7a28..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 55025508e40..8a53f0cbb42 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -34,6 +34,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..77c4136efda 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -165,6 +165,7 @@ from litellm.types.rerank import RerankResponse from litellm.types.router import RouterRateLimitError from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -1158,6 +1159,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 6c0e463bd23..75b0329315f 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -207,7 +208,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -218,7 +219,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..8bb6072d0a9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index da734cf8189..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: mapped: Final = super().map_openai_params_for_deployment( response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..05440d70017 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -117,6 +117,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,11 +1193,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index dd721e059a9..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1331,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9700465ebc3..9e125db4211 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -6275,9 +6279,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 903e6177e7b..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -55,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -417,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -464,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -494,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -512,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -520,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: diff --git a/litellm/main.py b/litellm/main.py index 5a1f2afdb10..1cf1a347407 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -156,6 +156,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -1297,6 +1298,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -5718,6 +5729,7 @@ def completion( model = deployment_id custom_llm_provider = "azure" routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5725,16 +5737,23 @@ def completion( api_key=api_key, litellm_params=supplemental_params.params, ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) drop_unsupported_reasoning: Final = ( litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True @@ -5830,7 +5849,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50b3560f05b..aa00ea73f8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5bed6037fa4..11e1602e9f8 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,6 +90,7 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, CustomPricingLiteLLMParams, @@ -1580,6 +1581,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 266a6a20131..9c3329d6387 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,10 +263,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index aeb25f81841..a656d8ebca3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -3501,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. @@ -3815,7 +3852,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3831,10 +3868,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3842,6 +3881,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -5107,6 +5156,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5196,19 +5285,30 @@ def get_optional_params( verbose_logger.debug( "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) ) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -6724,6 +6824,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6757,7 +6859,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6781,7 +6883,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50b3560f05b..aa00ea73f8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3070ebb0235..ed7dd5ec146 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -378,6 +378,20 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +@pytest.fixture +def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable[[str, bool], None]: + def declare(model_cost_key: str, supported: bool) -> None: + entry: Final = { + "litellm_provider": model_cost_key.split("/", 1)[0], + "mode": "chat", + "supports_mid_conversation_system": supported, + } + monkeypatch.setitem(litellm.model_cost, model_cost_key, entry) + litellm.get_model_info.cache_clear() + + return declare + + NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None diff --git a/tests/unit/integrations/websearch_interception/conftest.py b/tests/unit/integrations/websearch_interception/conftest.py new file mode 100644 index 00000000000..a9f79a60bfe --- /dev/null +++ b/tests/unit/integrations/websearch_interception/conftest.py @@ -0,0 +1,18 @@ +from typing import Final + +import pytest +import respx + +import litellm + +PERPLEXITY_SEARCH_URL: Final = "https://api.perplexity.ai/search" + + +@pytest.fixture +def perplexity_search(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> respx.Route: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "test-perplexity-key") + monkeypatch.delenv("PERPLEXITY_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return respx_mock.post(PERPLEXITY_SEARCH_URL).respond( + json={"results": [{"title": "AI news", "url": "https://example.com/ai-news", "snippet": "AI news."}]} + ) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py index 21e50561f57..fc415374096 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py @@ -5,15 +5,19 @@ Tests the end-to-end flow of websearch_interception callback with litellm.acompletion() for transparent server-side web search execution. """ -from unittest.mock import MagicMock +import datetime +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx import litellm from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) -from litellm.types.utils import LlmProviders, ModelResponse +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.utils import CallTypes, LlmProviders, ModelResponse @pytest.fixture @@ -378,7 +382,6 @@ async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): optional_params into the follow-up params without explicit removal causes the model to call the search tool again instead of synthesizing an answer. """ - from unittest.mock import patch websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) @@ -422,3 +425,54 @@ async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): if __name__ == "__main__": # Run with: pytest test_websearch_chat_completion.py -v -s pytest.main([__file__, "-v", "-s"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model,handler_model,custom_llm_provider", + [ + ("openrouter/openrouter/free", "openrouter/free", "openrouter"), + ("bedrock/converse_like/my-endpoint-model", "my-endpoint-model", "bedrock"), + ], +) +async def test_chat_followup_reuses_the_model_the_caller_requested( + requested_model: str, handler_model: str, custom_llm_provider: str, perplexity_search: respx.Route +) -> None: + """The provider handler sees the model after its routing prefixes are stripped, which + re-qualifying with the provider alone cannot restore.""" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.completion.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-chat-followup", + function_id="fn-chat-followup", + ) + litellm.completion( + model=requested_model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + logger: Final = WebSearchInterceptionLogger() + + plan: Final = await logger.async_build_chat_completion_agentic_loop_plan( + tools={ + "tool_calls": [{"id": "call_abc", "name": "litellm_web_search", "input": {"query": "latest ai news"}}], + "tool_type": "websearch", + "provider": custom_llm_provider, + "response_format": "openai", + }, + model=handler_model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=MagicMock(), + optional_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={"custom_llm_provider": custom_llm_provider}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + assert perplexity_search.call_count == 1 diff --git a/tests/unit/integrations/websearch_interception/test_websearch_responses.py b/tests/unit/integrations/websearch_interception/test_websearch_responses.py index 00b7299cc31..6ca812a5e40 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_responses.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_responses.py @@ -6,17 +6,22 @@ tool calls returned by /v1/responses, executes the search server-side, and builds a Responses-format follow-up request. """ +import datetime from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +import litellm + from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.types.integrations.custom_logger import ( RESPONSES_AGENTIC_SURFACE, ) +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import CallTypes, LlmProviders @@ -252,6 +257,112 @@ async def test_build_responses_plan_produces_responses_input(): assert patch_obj.model == "openai/gpt-4o" +_WEB_SEARCH_FUNCTION_CALL: Final = { + "tool_calls": [ + { + "id": "fc_1", + "call_id": "fc_1", + "type": "function_call", + "name": "litellm_web_search", + "arguments": '{"query": "latest ai news"}', + "input": {"query": "latest ai news"}, + } + ], + "tool_type": "websearch", + "response_format": "responses", +} + + +async def _responses_followup_model(model: str, custom_llm_provider: str, logging_obj: Logging | None) -> str | None: + logger: Final = WebSearchInterceptionLogger() + plan: Final = await logger.async_build_responses_agentic_loop_plan( + tools={**_WEB_SEARCH_FUNCTION_CALL, "provider": custom_llm_provider}, + model=model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=_responses_output_with_web_search(), + optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}]}, + logging_obj=logging_obj, + stream=False, + kwargs={ + "custom_llm_provider": custom_llm_provider, + "_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE, + }, + ) + assert plan.request_patch is not None + return plan.request_patch.model + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_build_responses_plan_keeps_the_provider_on_a_sub_path_model() -> None: + """#38829 on the Responses surface: a provider-stripped model whose remainder still holds a + slash was left unqualified, so the follow-up could not resolve a provider.""" + followup_model: Final = await _responses_followup_model("mantle/anthropic.claude-sonnet-5", "bedrock", None) + + assert followup_model == "bedrock/mantle/anthropic.claude-sonnet-5" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_responses_followup_reuses_the_model_the_caller_requested() -> None: + """An OpenRouter model whose own id starts with "openrouter/" reaches the handler as + "openrouter/free", which re-qualifying alone cannot tell apart from a request for model "free".""" + requested_model: Final = "openrouter/openrouter/free" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-responses-followup", + function_id="fn-responses-followup", + ) + litellm.responses( + model=requested_model, + input="What's the latest AI news?", + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + + followup_model: Final = await _responses_followup_model("openrouter/free", "openrouter", logging_obj) + + assert followup_model == requested_model + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_responses_followup_keeps_the_caller_model_across_the_chat_completions_bridge() -> None: + """A Responses request bridged to chat completions re-enters completion() with the stripped model.""" + requested_model: Final = "openrouter/openrouter/free" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-bridged-followup", + function_id="fn-bridged-followup", + ) + litellm.responses( + model=requested_model, + input="What's the latest AI news?", + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + litellm.completion( + model="openrouter/free", + custom_llm_provider="openrouter", + messages=[{"role": "user", "content": "What's the latest AI news?"}], + mock_response="searching", + litellm_logging_obj=logging_obj, + _skip_responses_api_bridge=True, + ) + + followup_model: Final = await _responses_followup_model("openrouter/free", "openrouter", logging_obj) + + assert followup_model == requested_model + + @pytest.mark.asyncio async def test_deployment_hook_converts_native_responses_web_search_tool(): """async_pre_call_deployment_hook converts a native Responses web_search tool diff --git a/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py b/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py index 72149e8a435..1020848ec89 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py @@ -239,6 +239,7 @@ class TestCallSiteWiring: tool_calls=tool_calls, optional_params={}, kwargs={}, + logging_obj=None, ) call_kwargs = mock_asearch.await_args.kwargs diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 3cbc3591009..6167e91c37f 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -364,6 +364,49 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert logger.cleanup_calls == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model,custom_llm_provider,routed_model", + [ + ("cohere/command-a-03-2025", "cohere_chat", "command-a-03-2025"), + ("anthropic/claude-2", "anthropic_text", "claude-2"), + ], +) +async def test_sdk_followup_keeps_a_model_whose_prefix_already_routes_to_its_provider( + model: str, custom_llm_provider: str, routed_model: str +) -> None: + plan: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages()), + ) + gate: Final = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]}) + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-prefix", + function_id="fn-prefix", + dynamic_success_callbacks=[gate], + ) + + followup: Final = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={"mock_response": "done"}, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + stream=False, + ) + + assert isinstance(followup, ModelResponse) + assert followup.model == routed_model + assert followup._hidden_params["custom_llm_provider"] == custom_llm_provider + + @pytest.mark.asyncio async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs( restore_callbacks, diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 66a8e6bbda6..a4a3ae285b1 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -26,6 +26,8 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, preserve_upstream_non_openai_attributes, process_response_headers, + qualify_agentic_followup_model, + qualify_provider_stripped_model, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -637,6 +639,87 @@ class TestProviderResponseHeadersInHiddenParams: assert "additional_headers" not in sibling._hidden_params +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("mantle/anthropic.claude-sonnet-5", "bedrock", "bedrock/mantle/anthropic.claude-sonnet-5"), + ("invoke/anthropic.claude-v2", "bedrock", "bedrock/invoke/anthropic.claude-v2"), + ("openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), + ("gpt-4o", "openai", "openai/gpt-4o"), + ("claude-sonnet-4-5", "anthropic", "anthropic/claude-sonnet-4-5"), + ], +) +def test_the_provider_prefix_is_restored(model: str, provider: str, expected: str) -> None: + assert qualify_provider_stripped_model(model, provider) == expected + + +@pytest.mark.parametrize( + "model,provider", + [ + ("bedrock/mantle/anthropic.claude-sonnet-5", "bedrock"), + ("openai/gpt-4o", "openai"), + ("cohere/command-a-03-2025", "cohere_chat"), + ("ai21/jamba-1.5-large", "ai21_chat"), + ("anthropic/claude-2", "anthropic_text"), + ], +) +def test_an_already_qualified_model_is_left_alone(model: str, provider: str) -> None: + assert qualify_provider_stripped_model(model, provider) == model + + +def test_a_provider_that_only_shares_a_prefix_is_still_qualified() -> None: + assert qualify_provider_stripped_model("openai_like/foo", "openai") == "openai/openai_like/foo" + + +def test_no_provider_leaves_the_model_untouched() -> None: + assert qualify_provider_stripped_model("gpt-4o", "") == "gpt-4o" + + +def test_a_cross_provider_patched_model_is_dispatched_as_the_hook_asked() -> None: + assert qualify_agentic_followup_model("anthropic/claude-sonnet-4-5", "gpt-4o", "openai") == ( + "anthropic/claude-sonnet-4-5" + ) + + +def test_a_bare_patched_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("gpt-4o-mini", "gpt-4o", "openai") == "openai/gpt-4o-mini" + + +def test_a_sub_path_request_model_keeps_its_provider() -> None: + assert qualify_agentic_followup_model(None, "mantle/anthropic.claude-sonnet-5", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_an_unpatched_ordinary_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model(None, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_model_already_holding_the_request_provider_is_left_alone() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "gpt-4o", "openai") == "openai/gpt-4o" + + +@pytest.mark.parametrize("patched", ["", None]) +def test_a_patch_without_a_model_falls_back_to_the_request_model(patched: str | None) -> None: + assert qualify_agentic_followup_model(patched, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_sub_path_model_still_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("mantle/anthropic.claude-sonnet-5", "gpt-4o", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_a_patched_model_on_a_json_configured_provider_reaches_that_provider() -> None: + assert qualify_agentic_followup_model("aihubmix/gpt-4o", "gpt-4o", "openai") == "aihubmix/gpt-4o" + + +def test_a_patch_echoing_the_request_model_keeps_the_request_provider() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "openai/gpt-4o", "openrouter") == ( + "openrouter/openai/gpt-4o" + ) + + def test_spend_metadata_merge_survives_the_logging_thread_inserting_into_the_request_metadata() -> None: metadata: Final[dict[str, object]] = {} diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 53df6610913..f6dae2c304f 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -783,3 +783,26 @@ def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mo assert request.url.params["api-version"] == "2024-12-01-preview" assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} assert response.choices[0].message.content == "Hello" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("config", [AzureOpenAIConfig(), AzureOpenAIGPT5Config()]) +def test_azure_keeps_a_later_developer_message_in_place_as_system(config: AzureOpenAIConfig) -> None: + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="my-deployment", + custom_llm_provider="azure", + api_base="https://example-resource.openai.azure.com", + model_info=None, + ) + assert [message["role"] for message in messages] == ["system", "user", "assistant", "system", "user"] + assert messages[3]["content"] == "Answer with exactly one word." diff --git a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index a82dd98e31d..2d9381ec16a 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Mapping from typing import Final @@ -279,6 +280,29 @@ def test_azure_gpt5_reasoning_effort_none_dropped(config: AzureOpenAIGPT5Config) assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" +@pytest.mark.parametrize(("model", "names_base_model"), [("azure/litellm-qa-sol-alias", True), ("azure/gpt-5", False)]) +def test_azure_dropped_effort_warning_suggests_base_model_only_for_a_name_the_map_lacks( + config: AzureOpenAIGPT5Config, caplog: pytest.LogCaptureFixture, model: str, names_base_model: bool +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model=model, + drop_params=True, + api_version="2025-04-01-preview", + declared_reasoning_efforts=(), + ) + + assert "reasoning_effort" not in params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort='none'" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.reasoning_effort_levels" in drop_warnings[0] + assert ("model_info.base_model" in drop_warnings[0]) is names_base_model + + # Logprobs support tests for Azure GPT-5.2 def test_azure_gpt5_2_supports_logprobs(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5.2 models support logprobs parameters. diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index ccac6ffc309..42dc222e420 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,5 +1,6 @@ import json import traceback +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -1098,3 +1099,25 @@ def test_map_azure_model_group(model_group_header, expected_model): config = AzureAICohereConfig() assert config._map_azure_model_group(model_group_header) == expected_model + + +def test_translate_developer_role_hoists_a_later_developer_message_for_a_model_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("azure_ai/system-first-model", False) + messages: Final = AzureAIStudioConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="system-first-model", + custom_llm_provider="azure_ai", + api_base="https://example.services.ai.azure.com/models", + model_info=None, + ) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user"] + assert messages[0]["content"] == "You are terse.\n\nAnswer with exactly one word." diff --git a/tests/unit/llms/base_llm/test_base_utils.py b/tests/unit/llms/base_llm/test_base_utils.py new file mode 100644 index 00000000000..7052fd04242 --- /dev/null +++ b/tests/unit/llms/base_llm/test_base_utils.py @@ -0,0 +1,494 @@ +from typing import Final + +from litellm.llms.base_llm.base_utils import ( + hoist_developer_messages_into_leading_system_message, + merge_system_message_runs, +) + + +class TestMergeSystemMessageRuns: + def test_later_developer_message_stays_in_place_while_the_leading_run_folds(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Sandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex\n\nSandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_run_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + + def test_hoisted_developer_message_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": 42}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": 42}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +class TestHoistDeveloperMessagesIntoLeadingSystemMessage: + def test_single_developer_message(self) -> None: + messages: Final = [ + {"role": "developer", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_developer_messages_merge(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Rule 1: Be concise."}, + {"role": "developer", "content": "Rule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rule 1: Be concise.\n\nRule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + + def test_system_followed_by_developer_merges(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base instructions."}, + {"role": "developer", "content": "Developer override."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base instructions.\n\nDeveloper override."}, + {"role": "user", "content": "Hello"}, + ] + + def test_developer_followed_by_system_merges(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Developer instruction."}, + {"role": "system", "content": "System instruction."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Developer instruction.\n\nSystem instruction."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_native_system_messages_merge(self) -> None: + messages: Final = [ + {"role": "system", "content": "System part 1."}, + {"role": "system", "content": "System part 2."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System part 1.\n\nSystem part 2."}, + {"role": "user", "content": "Hello"}, + ] + + def test_empty_string_side_does_not_pad(self) -> None: + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": ""}, {"role": "developer", "content": "Rules"}] + ) + ) == [{"role": "system", "content": "Rules"}] + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": "Rules"}, {"role": "developer", "content": ""}] + ) + ) == [{"role": "system", "content": "Rules"}] + + def test_empty_string_message_adds_no_block_when_the_run_merges_as_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": ""}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + + def test_two_cached_string_messages_keep_one_breakpoint_each(self) -> None: + messages: Final = [ + {"role": "system", "content": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"role": "system", "content": "B", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + }, + ] + + def test_cache_control_supplied_only_by_the_later_message_marks_only_its_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "A"}, + {"role": "developer", "content": "B", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A"}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + def test_cache_control_of_the_first_message_stays_on_its_own_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "developer", "content": "Additional instructions"}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAdditional instructions"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_cached_string_message_merged_with_block_content_keeps_its_breakpoint(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": [{"type": "text", "text": "Answer with exactly one word."}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_plain_messages_ahead_of_a_cached_one_keep_their_own_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "developer", "content": "Rule"}, + {"role": "developer", "content": "Cached", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Base"}, + {"type": "text", "text": "\n\nRule"}, + {"type": "text", "text": "\n\nCached", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_name_of_the_last_named_message_survives_a_long_run(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base", "name": "first"}, + *({"role": "developer", "content": f"Rule {index}"} for index in range(2000)), + {"role": "developer", "content": "Last", "name": "last"}, + {"role": "user", "content": "Hello"}, + ] + merged: Final = list(hoist_developer_messages_into_leading_system_message(messages)) + assert len(merged) == 2 + assert merged[0]["name"] == "last" + assert merged[0]["content"] == "\n\n".join(["Base", *(f"Rule {index}" for index in range(2000)), "Last"]) + + def test_billing_metadata_keeps_its_own_block_when_merged(self) -> None: + billing: Final = {"role": "system", "content": "x-anthropic-billing-header: cc_version=1.0;"} + advisory: Final = {"role": "system", "content": "Guardrail advisory: request was flagged"} + assert list(hoist_developer_messages_into_leading_system_message([billing, advisory])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + {"type": "text", "text": "\n\nGuardrail advisory: request was flagged"}, + ], + } + ] + assert list(hoist_developer_messages_into_leading_system_message([advisory, billing])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Guardrail advisory: request was flagged"}, + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + ], + } + ] + + def test_developer_message_after_a_user_turn_is_hoisted_into_the_leading_system_message(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "developer", "content": "Developer turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System turn 1\n\nDeveloper turn 2"}, + {"role": "user", "content": "User turn 1"}, + {"role": "user", "content": "User turn 2"}, + ] + + def test_hoisted_developer_message_keeps_only_the_fields_a_system_message_has(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word.", "name": "codex", "extra_key": "x"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word.", "name": "codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_developer_message_in_second_position_without_a_leading_system_message_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_every_later_developer_message_is_hoisted_in_order_across_assistant_and_tool_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "developer", "content": "Update B"}, + {"role": "user", "content": "Turn 3"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A\n\nUpdate B"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "user", "content": "Turn 3"}, + ] + + def test_developer_message_that_closes_the_conversation_after_an_assistant_turn_becomes_a_user_turn(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with the single word PONG and nothing else."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "user", "content": "Reply with the single word PONG and nothing else."}, + ] + + def test_closing_developer_message_keeps_its_cache_mark_as_a_user_turn(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with PONG.", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages))[-1] == { + "role": "user", + "content": "Reply with PONG.", + "cache_control": {"type": "ephemeral"}, + } + + def test_only_the_closing_developer_run_after_an_assistant_turn_becomes_user_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "developer", "content": "Closing B"}, + {"role": "developer", "content": "Closing C"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A"}, + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "user", "content": "Closing B"}, + {"role": "user", "content": "Closing C"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_user_turn_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Reply with exactly one word: the capital of France"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Reply with exactly one word: the capital of France"}, + {"role": "user", "content": "Hi there"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_tool_result_is_hoisted(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + {"role": "developer", "content": "Answer with exactly one word."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + ] + + def test_hoisted_single_text_block_developer_message_joins_string_instructions_as_text(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "developer", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex\n\n"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + + def test_multi_block_developer_message_keeps_its_blocks_after_string_instructions(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi"}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Codex"}, + {"type": "text", "text": "\n\nRule 1"}, + {"type": "text", "text": "Rule 2"}, + ], + }, + {"role": "user", "content": "Hi"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_system_message_without_content_folds_into_the_run_instead_of_raising(self) -> None: + messages: Final = [ + {"role": "system", "content": "Rules"}, + {"role": "system"}, + {"role": "system", "content": None}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1"}, + {"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "system", "content": "System turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == messages + + def test_list_content_merge_preserves_block_level_cache_control(self) -> None: + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}]}, + {"role": "developer", "content": [{"type": "text", "text": "Extra"}]}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nExtra"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..2d75ce1f22f 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -966,7 +967,9 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode "us.openai.gpt-6-astra", ], ) -def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, local_model_cost_map): +def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( + model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: config = AmazonConverseConfig() assert litellm.utils.is_explicitly_disabled_factory( model=model, custom_llm_provider="bedrock_converse", key="supports_minimal_reasoning_effort" @@ -980,15 +983,19 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, drop_params=False, ) - optional_params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model=model, - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=True, + ) _, additional_request_params, _, _ = config._prepare_request_params(optional_params, model) assert "reasoning" not in additional_request_params assert "thinking" not in additional_request_params + dropped: Final = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] @pytest.mark.parametrize( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..a1466cfdae4 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -7,13 +7,17 @@ falls back to the Chat Completions bridge and rides Converse. import json import logging from importlib.resources import files +from typing import Final from unittest.mock import patch import pytest import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses -from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig +from litellm.llms.bedrock.responses.transformation import ( + BedrockOpenAIResponsesConfig, + _without_disabled_reasoning_effort, +) from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -337,13 +341,37 @@ class TestDisabledReasoningEffort: ) @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) - def test_effort_level_disabled_in_model_map_is_dropped_with_drop_params(self, model, local_model_cost_map): - params = _cfg().map_openai_params( - response_api_optional_params={"reasoning": {"effort": "minimal", "summary": "auto"}, "max_output_tokens": 64}, - model=model, - drop_params=True, - ) + def test_effort_level_disabled_in_model_map_is_dropped_with_a_warning( + self, model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = _cfg().map_openai_params( + response_api_optional_params={ + "reasoning": {"effort": "minimal", "summary": "auto"}, + "max_output_tokens": 64, + }, + model=model, + drop_params=True, + ) assert params == {"reasoning": {"summary": "auto"}, "max_output_tokens": 64} + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] + assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] + + def test_bedrock_disabled_effort_drop_warns_on_its_own( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = _without_disabled_reasoning_effort( + {"reasoning": {"effort": "minimal"}, "max_output_tokens": 64}, MODEL, True + ) + assert params == {"max_output_tokens": 64} + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] + assert len(dropped) == 1 and MODEL in dropped[0] def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): params = _cfg().map_openai_params( diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a2289167ad4..0ae6a73968d 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -9,6 +9,7 @@ gate, the URL construction for both paths, and the shared Bearer auth. import copy import logging +from typing import Final import pytest from botocore.exceptions import ( @@ -572,6 +573,32 @@ class TestBedrockMantleServiceTier: assert "priority" in str(mock_warning.call_args) +class TestBedrockMantleRefusedEffort: + def test_dropping_a_refused_effort_does_not_suggest_declaring_levels_mantle_ignores( + self, local_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = BedrockMantleResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused}}, + model="openai.gpt-6.1-sol", + drop_params=True, + ) + + assert "reasoning" not in params + drop_warnings: Final = [ + record.getMessage() + for record in caplog.records + if f"Dropping reasoning.effort={refused!r}" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info" not in drop_warnings[0] + + class TestBedrockMantleReasoningSummary: @pytest.mark.parametrize("summary", ["concise", "detailed"]) def test_unsupported_reasoning_summary_dropped_when_drop_params_true(self, summary): diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 90d1f6772a0..d8326536d03 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5634,6 +5634,63 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers) +class TestAgenticFollowUpKeepsTheProviderPrefix: + @staticmethod + async def _run_followup( + model: str, custom_llm_provider: str, patched_model: str | None = None + ) -> ModelResponse: + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, + ) + + response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "hi"}], model=patched_model + ), + ), + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"mock_response": "ok from followup"}, + kwargs={}, + custom_llm_provider=custom_llm_provider, + depth=0, + max_loops=2, + fingerprints=[], + fingerprint="fp", + ) + assert isinstance(response, ModelResponse) + return response + + @pytest.mark.asyncio + async def test_a_sub_path_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_an_ordinary_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o-mini", "openai") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5") + + assert response.model == "claude-sonnet-4-5" + assert response._hidden_params["custom_llm_provider"] == "anthropic" + + @pytest.mark.asyncio + async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None: + response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") + + assert response._hidden_params["custom_llm_provider"] == "openrouter" + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 61e5545b8bf..28f4fdead9e 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -2358,3 +2359,86 @@ def test_global_disable_flag_with_transform_messages_helper( ) assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] + + +_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +def test_translate_developer_role_keeps_a_later_developer_message_in_place_for_an_undeclared_fireworks_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +def test_translate_developer_role_leaves_a_leading_system_and_developer_pair_unmerged_for_an_undeclared_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("model", ["qwen3p8-max", "accounts/fireworks/models/qwen3p8-max"]) +def test_translate_developer_role_hoists_a_later_developer_message_for_the_system_first_qwen3p8_model( + local_model_cost_map: None, model: str +) -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model=model, + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("deployment_declared", "expected_roles"), + [(False, ["system", "user", "user"]), (True, ["system", "user", "system", "user"])], +) +def test_translate_developer_role_follows_the_deployment_model_info_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], deployment_declared: bool, expected_roles: list[str] +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", not deployment_declared) + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/shared-backend", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info={"id": "deployment-a", "supports_mid_conversation_system": deployment_declared}, + ) + + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 878e0285cd1..7fc6ac2c436 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -1,5 +1,5 @@ import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final, TypedDict, cast from unittest.mock import MagicMock, patch @@ -7,6 +7,7 @@ from urllib.parse import quote import httpx import pytest +import respx from openai.types.responses import ( EasyInputMessage, ResponseFunctionToolCall, @@ -81,6 +82,14 @@ def _mock_http_client(response_body: Mapping[str, object]) -> MagicMock: return client +def _fireworks_responses_route(model: str) -> respx.Route: + return respx.post(FIREWORKS_RESPONSES_URL).mock(return_value=httpx.Response(200, json=_fireworks_response(model))) + + +def _routed_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + def _sent_request(client: MagicMock) -> tuple[str, Mapping[str, str], Mapping[str, object]]: kwargs: Final = client.post.call_args.kwargs body: Final = kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) @@ -182,6 +191,60 @@ def test_responses_call_folds_developer_items_into_instructions() -> None: ) +@respx.mock +def test_responses_call_keeps_developer_items_in_place_for_a_deployment_that_accepts_later_system_items() -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/accepts-later-system") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/accepts-later-system", + instructions="You are Codex", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model_info={"id": "deployment-1", "supports_mid_conversation_system": True}, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "You are Codex" + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, + {"role": "user", "content": "What is the capital of France?"}, + ) + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "closing_role"), + [(None, "system"), ({"id": "deployment-1", "supports_mid_conversation_system": False}, "user")], +) +def test_responses_call_sends_the_closing_developer_item_as_a_user_turn_only_for_a_declared_system_first_deployment( + model_info: Mapping[str, object] | None, closing_role: str +) -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/kimi-k3") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Reply with PONG."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "Answer with exactly one word." + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": closing_role, "content": "Reply with PONG.", "type": "message"}, + ) + + def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): @@ -667,3 +730,40 @@ def test_responses_call_forwards_litellm_user_id_as_user( _, _, body = _sent_request(client) assert body.get("user") == expected_user assert "fireworks_forward_user_id" not in body + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "expected_instructions", "expected_input"), + [ + (None, "Answer in one word.", ({"role": "user", "content": "What is the capital of France?"},)), + ( + {"id": "deployment-1", "supports_mid_conversation_system": True}, + None, + ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Answer in one word.", "type": "message"}, + ), + ), + ], +) +def test_responses_call_forwards_developer_items_in_place_only_when_the_deployment_itself_declares_true( + declare_mid_conversation_system: Callable[[str, bool], None], + model_info: Mapping[str, object] | None, + expected_instructions: str | None, + expected_input: tuple[Mapping[str, object], ...], +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", True) + route: Final = _fireworks_responses_route("accounts/fireworks/models/shared-backend") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/shared-backend", + input=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "developer", "content": "Answer in one word."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body.get("instructions") == expected_instructions + assert tuple(body["input"]) == expected_input diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..2f63ef070e6 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,9 +8,13 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm @@ -183,3 +187,92 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +_LATER_DEVELOPER_INPUT: Final = ( + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +_VLLM_RESPONSES_URL: Final = "https://test-vllm.example.com/v1/responses" + + +def _vllm_responses_route(output_text: str) -> respx.Route: + return respx.post(_VLLM_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_make_mock_responses_api_response(output_text)) + ) + + +def _sent_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + +@respx.mock +def test_hosted_vllm_responses_folds_a_later_developer_item_into_instructions_for_a_system_first_model( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("hosted_vllm/Qwen/Qwen3.6-27B", False) + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3.6-27B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == (_LATER_DEVELOPER_INPUT[0], _LATER_DEVELOPER_INPUT[2]) + + +@respx.mock +def test_hosted_vllm_responses_folds_for_a_deployment_declared_system_first_and_sends_a_closing_developer_as_user() -> None: + assistant_turn: Final = {"type": "message", "role": "assistant", "content": "Paris"} + route: Final = _vllm_responses_route("PONG") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=[*_LATER_DEVELOPER_INPUT, assistant_turn, {"role": "developer", "content": "Reply with PONG."}], + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == ( + _LATER_DEVELOPER_INPUT[0], + _LATER_DEVELOPER_INPUT[2], + assistant_turn, + {"role": "user", "content": "Reply with PONG.", "type": "message"}, + ) + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_later_developer_item_in_place_for_an_undeclared_model() -> None: + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant." + assert tuple(body["input"]) == _LATER_DEVELOPER_INPUT + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_developer_item_without_content_on_a_deployment_declared_system_first() -> None: + route: Final = _vllm_responses_route("Paris") + input_items: Final = ( + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"type": "message", "role": "developer"}, + ) + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input=list(input_items), + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + assert tuple(_sent_body(route)["input"]) == input_items diff --git a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 9b8788d3769..8a690a70a97 100644 --- a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -639,3 +639,25 @@ def test_litellm_gateway_from_sdk_with_thinking_param(respx_mock): ) e = exc_info.value assert "Connection error." in str(e) + + +def test_translate_developer_role_leaves_a_later_developer_message_in_place_for_the_downstream_proxy() -> None: + translated: Final = LiteLLMProxyChatConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="fireworks-qwen", + custom_llm_provider="litellm_proxy", + api_base="http://inner-proxy:4000", + model_info=None, + ) + + assert list(translated) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 70436662ef0..5b85a94b24d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,7 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ import base64 +import copy import json +from collections.abc import Callable from typing import Final import pytest @@ -1820,3 +1822,97 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: ) assert request["tools"][0] is tool + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("config", "custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + (OpenAIGPTConfig(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPT5Config(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", False, ["system", "user", "assistant", "user"]), + (OpenAIGPTConfig(), "groq", None, None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "groq", None, False, ["system", "user", "assistant", "user"]), + ], +) +def test_translate_developer_role_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + config: OpenAIGPTConfig, + custom_llm_provider: str, + api_base: str | None, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles + assert "Answer with exactly one word." in [message["content"] for message in messages if message["role"] == "system"][-1] + + +@pytest.mark.parametrize("role", [["user"], {"name": "user"}]) +@pytest.mark.parametrize( + ("declared", "expected_order"), + [(None, ("caller", "instruction")), (False, ("instruction", "caller"))], +) +def test_translate_developer_role_places_a_message_with_a_non_string_role_like_any_conversation_turn( + declare_mid_conversation_system: Callable[[str, bool], None], + role: object, + declared: bool | None, + expected_order: tuple[str, str], +) -> None: + if declared is not None: + declare_mid_conversation_system("groq/system-first-model", declared) + caller_turn: Final = {"role": role, "content": "Hi there"} + by_name: Final = {"caller": caller_turn, "instruction": {"role": "system", "content": "Answer in one word."}} + messages: Final = OpenAIGPTConfig().translate_developer_role_to_system_role( + messages=[caller_turn, {"role": "developer", "content": "Answer in one word."}], + model="system-first-model", + custom_llm_provider="groq", + api_base=None, + model_info=None, + ) + assert list(messages) == [by_name[name] for name in expected_order] + + +@pytest.mark.respx(assert_all_called=True) +def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_system_first_deployment( + respx_mock: respx.MockRouter, +) -> None: + chat: Final = respx_mock.post("http://vllm.internal:8000/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + messages: Final = [ + {"role": "user", "content": "Describe the image."}, + {"role": "assistant", "content": "Send it."}, + { + "role": "developer", + "content": [{"type": "text", "text": "Look at this."}, {"type": "image_url", "image_url": "data:image/png;base64,aW1hZ2U="}], + }, + ] + sent: Final = copy.deepcopy(messages) + litellm.completion( + model="openai/system-first-model", + api_base="http://vllm.internal:8000/v1", + api_key="sk-test", + messages=messages, + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + ) + assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" + assert messages == sent diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index a185cdaf718..f5a33f87fd0 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, cast @@ -19,6 +20,7 @@ from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponse from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, IncompleteDetails, @@ -2479,6 +2481,105 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert mapped["reasoning"] == {"summary": "auto"} + @pytest.mark.parametrize( + ("declared_reasoning_efforts", "suggests_declared_levels"), + [((), True), (None, False)], + ids=["responses route reads model_info levels", "compact route reads none"], + ) + def test_dropping_a_refused_effort_logs_one_warning_naming_only_the_fix_its_route_honors( + self, + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + declared_reasoning_efforts: tuple[str, ...] | None, + suggests_declared_levels: bool, + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="gpt-6.1-sol", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + response_api_optional_params={"reasoning": {"effort": refused}}, + drop_params=True, + declared_reasoning_efforts=declared_reasoning_efforts, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert repr(refused) in drop_warnings[0] and "gpt-6.1-sol" in drop_warnings[0] + assert ("model_info.reasoning_effort_levels" in drop_warnings[0]) is suggests_declared_levels + assert "model_info.base_model" not in drop_warnings[0] + + def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) + + def test_turning_reasoning_off_for_a_model_without_supports_reasoning_drops_it_silently( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none", "summary": "auto"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + + def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + litellm.register_model( + { + "pricing-only-test-model": { + "litellm_provider": "openai", + "mode": "responses", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none"}}, + model="pricing-only-test-model", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "pricing-only-test-model")) + + def test_a_forwarded_effort_logs_no_drop_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert mapped["reasoning"] == {"effort": "medium"} + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): """The gate turns a stale false into a local refusal, so the Responses-only codex row has to diff --git a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py index 4cdc674c774..807a43c5d9d 100644 --- a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py +++ b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py @@ -1,3 +1,4 @@ +import logging import re from collections.abc import Mapping from typing import Final @@ -1589,3 +1590,55 @@ def test_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert "max_tokens" not in params assert params["reasoning_effort"] == "max" assert params["verbosity"] == "low" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "level", "advice"), + [ + ("azure", "none", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("azure", "xhigh", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("openai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("azure_ai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("github_copilot", "xhigh", frozenset()), + ], +) +def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_only_the_fixes_its_route_honors( + caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str, advice: frozenset[str] +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider=custom_llm_provider, + reasoning_effort=level, + drop_params=True, + ) + + assert "reasoning_effort" not in params + drop_warnings: Final = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert repr(level) in drop_warnings[0] and "gpt-5.6-terra-ptu" in drop_warnings[0] + assert ( + frozenset( + fragment + for fragment in ("model_info.reasoning_effort_levels", "model_info.base_model") + if fragment in drop_warnings[0] + ) + == advice + ) + + +def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider="azure", + reasoning_effort="medium", + drop_params=True, + ) + + assert params["reasoning_effort"] == "medium" + assert not [record for record in caplog.records if "reasoning_effort" in record.getMessage()] diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index 121fab66c15..e78e1a6f12b 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -181,3 +181,20 @@ def test_litellm_responses(): print("response: ", response) assert isinstance(response.usage.completion_tokens_details, CompletionTokensDetails) + + +def test_translate_developer_role_keeps_developer_messages_untouched() -> None: + messages: Final = [ + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert ( + OpenAIOSeriesConfig().translate_developer_role_to_system_role( + messages=messages, + model="my-reasoning-deployment", + custom_llm_provider="openai", + api_base=None, + model_info=None, + ) + == messages + ) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 11d7c7d6d3c..5b27a446fb3 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2,7 +2,7 @@ import asyncio import importlib import json import os -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncCursorPage, MessageData, OpenAIChatCompletion, + OpenAIConfig, OpenAIMessage as Message, Run, SyncCursorPage, @@ -2010,3 +2011,40 @@ async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_head assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} assert sent.headers["x-caller"] == "unit" assert sent.headers["authorization"] == "Bearer sk-test" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + ("openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", None, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", False, ["system", "user", "assistant", "user"]), + ], +) +def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + custom_llm_provider: str, + api_base: str, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = OpenAIConfig().translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py index feb7090c2e3..cc5ad10b645 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py @@ -15,16 +15,20 @@ Tests cover: - fail_closed raises HTTPException; fail_open forwards uncompressed """ +import datetime import hashlib import json from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import httpx import pytest from fastapi import HTTPException +import litellm from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( COMPRESR_RETRIEVE_TOOL_NAME, CompresrGuardrail, @@ -33,7 +37,7 @@ from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( _scoped_store_key, has_compresr_retrieve_tool, ) -from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs FAKE_API_BASE = "https://compresr.example.com" FAKE_API_KEY = "cmp_test-key" @@ -1435,6 +1439,57 @@ async def test_agentic_plan_builds_responses_followup_shape( assert output_item["output"] == TOOL_OUTPUT +@pytest.mark.asyncio +async def test_responses_retrieval_followup_reuses_the_model_the_caller_requested( + guardrail: CompresrGuardrail, +) -> None: + requested_model: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" + hash_value: Final = "f" * 24 + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-1", + function_id="fn-compresr-followup", + ) + litellm.responses( + model=requested_model, + input=USER_QUESTION, + mock_response="retrieving", + litellm_logging_obj=logging_obj, + ) + guardrail._store_originals(_scoped_store_key(logging_obj), {hash_value: TOOL_OUTPUT}) + response: Final = MagicMock() + response.output = [{"type": "function_call", "call_id": "fc_1"}] + response.content = None + + plan: Final = await guardrail.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "fc_1", + "type": "function", + "name": COMPRESR_RETRIEVE_TOOL_NAME, + "arguments": {"hash": hash_value}, + } + ] + }, + model="converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": USER_QUESTION}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + + @pytest.mark.asyncio async def test_agentic_plan_chat_parallel_tool_calls_echoes_only_retrieve( guardrail: CompresrGuardrail, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 3336b20f5eb..741fea5f2b9 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -21,6 +21,7 @@ Tests cover: loop resolves the retrieve tool call, then fake-streamed back to the client """ +import datetime import json import time from typing import Final, Optional @@ -33,6 +34,7 @@ from fastapi import HTTPException import litellm +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( HeadroomGuardrail, has_headroom_retrieve_tool, @@ -771,6 +773,72 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_ assert output_item["output"] == original_content +@pytest.mark.asyncio +async def test_responses_retrieval_followup_reuses_the_model_the_caller_requested( + guardrail: HeadroomGuardrail, +) -> None: + requested_model: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" + retrieve_hash: Final = "b573993006976af767214fac" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-1", + function_id="fn-headroom-followup", + ) + litellm.responses( + model=requested_model, + input=f"What does it say? hash={retrieve_hash}", + mock_response="retrieving", + litellm_logging_obj=logging_obj, + ) + response: Final = MagicMock() + response.choices = None + response.content = None + response.output = [ + { + "type": "function_call", + "id": "fc_abc123", + "call_id": "call_abc123", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": json.dumps({"hash": retrieve_hash}), + } + ] + guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({retrieve_hash}), time.monotonic() + 999) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response("The full compressed content."), + ): + plan: Final = await guardrail.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": retrieve_hash}, + } + ] + }, + model="converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": f"What does it say? hash={retrieve_hash}"}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + + @pytest.mark.asyncio async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messages( guardrail: HeadroomGuardrail, diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d0564812c48..1d8ee338cbb 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,8 +626,9 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "reasoning_effort unrecognized sent as None (rounding up) to deployment deployment-1, " - "model group kimi-k3, model hosted_vllm/my-kimi" + "Dropping reasoning level unrecognized (drop_params=True) for deployment deployment-1, model group " + "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " + "level. List every level the deployment supports." ] def test_line_breaks_in_logged_names_cannot_forge_a_log_line(self, caplog: pytest.LogCaptureFixture) -> None: @@ -669,6 +670,27 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } + def test_only_a_dropped_level_logs_a_warning(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + declared: Final = {"reasoning_effort_levels": ["low", "high"]} + + _sent("none", declared, drop_params=True) + _sent("medium") + _sent("default", declared, drop_params=True) + _sent("default", declared) + + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + logging.INFO, + logging.INFO, + ] + dropped: Final = caplog.records[0].getMessage() + assert "Dropping reasoning level none" in dropped + assert "model_info.reasoning_effort_levels" in dropped + class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: @pytest.mark.parametrize( diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..2870016a5e9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10869,6 +10869,44 @@ def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_e assert "cache_control_injection_points" not in optional_params +@pytest.mark.parametrize( + ("cost_map_declared", "deployment_declared", "expected"), + [ + (None, None, None), + (False, None, False), + (True, None, True), + (True, False, False), + (False, True, True), + (None, False, False), + ], +) +def test_declared_mid_conversation_system_support_prefers_the_deployment_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], + cost_map_declared: bool | None, + deployment_declared: bool | None, + expected: bool | None, +) -> None: + if cost_map_declared is not None: + declare_mid_conversation_system("hosted_vllm/mid-conversation-probe", cost_map_declared) + deployment_model_info: Final = ( + {"id": "deployment-1"} + if deployment_declared is None + else {"id": "deployment-1", "supports_mid_conversation_system": deployment_declared} + ) + assert ( + litellm.utils.declared_mid_conversation_system_support( + "mid-conversation-probe", "hosted_vllm", deployment_model_info + ) + is expected + ) + + +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") + litellm.get_model_info.cache_clear() + assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None + + def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy applies cache_control_injection_points itself, so the point must ride along in extra_body.""" diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 13a64577efb..05b776cb873 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -1,4 +1,5 @@ import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -2097,3 +2098,146 @@ def test_supports_system_message_merges_system_prompt_into_user_message() -> Non ] assert map_system_message_pt(messages=messages) == [{"role": "user", "content": "Listen here! Hello there!"}] + + +def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) + + +@pytest.mark.parametrize("provider", ["ollama", "ollama_chat", "lemonade", "huggingface"]) +def test_dropping_thinking_for_an_unmapped_local_model_sends_no_request( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + outbound: Final = respx.MockRouter(assert_all_mocked=False) + monkeypatch.setattr( + litellm, "module_level_client", HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(outbound.handler))) + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="unmapped-local-model", custom_llm_provider=provider, thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + assert len([record for record in caplog.records if "Dropping thinking" in record.getMessage()]) == 1 + assert not outbound.calls + + +def test_dropping_an_unsupported_thinking_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "model_info.base_model")) + + +@pytest.mark.parametrize( + ("reasoning_kwargs", "warns"), + [ + ({"reasoning_effort": "none"}, False), + ({"thinking": {"type": "disabled"}}, False), + ({"reasoning_effort": "high"}, True), + ({"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ], +) +def test_reasoning_drop_warns_unless_a_mapped_non_reasoning_model_was_asked_for_none( + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + reasoning_kwargs: dict[str, object], + warns: bool, +) -> None: + litellm.register_model( + { + "azure/plain-chat-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "supports_reasoning": False, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="plain-chat-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + assert any("Dropping" in record.getMessage() for record in caplog.records) is warns + + +@pytest.mark.parametrize("reasoning_kwargs", [{"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}]) +def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture, reasoning_kwargs: dict[str, object] +) -> None: + litellm.register_model( + { + "azure/pricing-only-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="pricing-only-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("pricing-only-test-deployment", "model_info.base_model")) + + +def test_dropping_reasoning_for_a_catalog_reasoning_model_does_not_suggest_base_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + assert litellm.model_cost["azure/gpt-5.6-sol"].get("supports_reasoning") is True + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-5.6-sol", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.base_model" not in drop_warnings[0] + + +def test_turning_reasoning_off_for_a_catalog_non_reasoning_model_drops_it_silently( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-4o", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + assert not [record for record in caplog.records if "Dropping" in record.getMessage()]