From 71babd931bf2bf1823f73be25c78bc35194d8c15 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:12:13 -0700 Subject: [PATCH 1/3] fix(llms): move later developer messages for models declared system-first (#48897) * fix(transformation): merge consecutive system and developer messages for non-OpenAI providers and Responses API * fix(base_utils): keep later cache_control and billing metadata boundaries when merging system messages * perf(base_utils): fold consecutive system messages in a single linear pass The tuple accumulator passed to reduce copied the whole prefix on every step, making map_developer_role_to_system_role O(n^2) in the number of messages. Message count is not bounded by default, so a caller could tie up a proxy worker with thousands of small messages before any provider call. Use a list and merge into its last element in place instead. * fix(base_utils): hoist later developer messages into the leading system message for non-OpenAI providers A developer message that arrives after the first user turn (Codex CLI's per-turn permissions item, a re-injected instruction after compaction, a mid-conversation policy update) was translated to a system message in place, and system-first chat templates such as Fireworks' qwen3.8 reject any system message that is not at the beginning with 400. The fold that merges consecutive system messages never saw it because a user turn sat in between. Move every developer message that follows the leading system block to the end of that block before the fold, so it merges into the single leading system message; client-authored mid-conversation system messages stay where the client put them. * fix(openai): hoist later developer messages into one leading system message on OpenAI-compatible backends Developer role translation moves onto the provider config. OpenAI's own endpoint, Azure OpenAI, and the o-series keep their current behavior. Every other config inheriting OpenAIGPTConfig (Fireworks, DeepSeek, Together, vLLM, Groq, openai/ with a custom api_base) hoists developer messages that arrive after the leading system block into that block and folds the block into a single system message, which is what system-first chat templates require. Providers with their own message format are untouched. Messages carrying cache_control or Anthropic billing metadata fold as their own text blocks, so breakpoints and provider-side stripping survive the merge. The Responses bridge goes back to prepending instructions as a separate system message and lets the hoist fold it. * refactor(base_utils): fold each run of system messages in one pass The pairwise fold recopied the accumulated system text or block list on every merge, so a request carrying many consecutive system or developer messages cost quadratic time in preprocessing. Group consecutive system messages once and build the merged message from the whole run: plain strings join with a blank line, and any cached, billing, or block content keeps one text block per message. The name of the last named message in the run survives, as before. * fix(base_utils): keep a closing developer message in place and hoist for OpenAIConfig backends A developer message that closes the conversation stays where the client put it, translated to system in place, so the request never ends on the assistant's turn. OpenAIConfig and its subclasses (Azure AI Foundry, GitHub Copilot, ChatGPT, RAGFlow) now share the OpenAI-compatible hoist, litellm_proxy passes developer messages through to the downstream proxy, and the Anthropic billing header literals use the shared constant * fix(base_utils): keep a closing developer message in place only after an assistant turn A developer message that closes the conversation after a user or tool message hoists like any other, so the request shape the ticket reports keeps working on system-first templates; only a closing run that follows an assistant turn stays in place * refactor(openai): reuse is_openai_backed_api_base for the hosted endpoint check * test(base_utils): cover an empty system message inside a block-content run Codecov flagged the empty-string branch of _text_blocks as the one line the PR left uncovered. The existing empty-string test only hits the plain-text join; this one forces the block path by merging an empty system message with a developer message that carries list content. * fix(openai): hoist developer messages for litellm_proxy and harden the system fold - drop the LiteLLMProxyChatConfig passthrough so litellm_proxy/ hoists later developer messages like every other OpenAI-compatible endpoint, whichever backend or LiteLLM version sits downstream - a system message with null or missing content inside a folded run is dropped instead of raising KeyError('content') - a message-level cache_control on a list-content member lands on its last block so the breakpoint survives the fold * fix(litellm_proxy): keep the AllMessageValues import the transform signatures still name * fix(fireworks_ai): hoist developer items after the first turn on the native Responses path The native Fireworks Responses path landed on staging mapping developer input items to system in place, so a developer item after the first input item still hit the same "System message must be at the beginning" 400 on system-first chat templates. The ordering rules now live in one shared helper over item roles, and the native path reorders through it before mapping, keeping a closing developer item after an assistant turn in place exactly like the chat path does * fix(transformation): carry the developer-message hoist onto main 963333d654 Resolves the main merge for this branch: the tests move to tests/unit, every translate_developer_role_to_system_role override takes and returns a Sequence, Azure OpenAI keeps the in-place mapping, and the system content reads tolerate a missing key * fix(transformation): join a lone plain text block as text when hoisting developer messages The Responses bridge turns every developer item into a one-block list, so the hoisted run merged the instructions string and that block as a block list, and Databricks could no longer collapse it into the single system string it sends today. A lone text block with no extra keys now joins the run as text, while multi-block, cached, and billing-header content keep the block path * fix(transformation): hoist later developer messages only for models that reject mid-conversation system messages Runs of consecutive system messages still fold into one, which keeps the cached prefix stable. A developer message after the first turn moves into the leading system message only when the model declares supports_mid_conversation_system false, or when it is a Fireworks model with no declaration, matching the native Fireworks Responses route. Every other backend keeps it in place as before, so their prompt cache prefix is unchanged. Malformed system content passes through unmerged so the provider returns its own 400. * fix(responses): fold later developer items into instructions on hosted_vllm for system-first models Moves the Responses developer fold shared by Fireworks into one module and applies it to hosted_vllm native Responses when the model declares supports_mid_conversation_system false. Renames the run-merging helper so it no longer shares a name with the Databricks one, and returns tuples from the block helpers. * fix(llms): place developer messages only for models declared system-first Models without a supports_mid_conversation_system declaration now keep the in-place developer-to-system translation they had before, so their cached prompt prefix does not change. Hoisting and folding run only when the deployment's model_info or the cost map declares the flag false, with the deployment value read first. A developer message that closes the conversation after an assistant turn is sent as a user message on those models instead of being hoisted, merged list content keeps a blank-line separator before each later part, and the cost map declares the flag false for fireworks_ai qwen3p8-max * fix(proxy): keep a closing developer message's cache mark when it is sent as a user turn * test: type the developer-hoist tests and fake HTTP with respx * fix: count non-string roles as turns, keep the Fireworks fold for twins, and copy closing developer content A message whose role is a list or dict now counts as a conversation turn instead of raising a 500. Fireworks native Responses skips the developer-item fold only when the deployment's own model_info declares supports_mid_conversation_system true, since the router registers every deployment's flag under its shared backend key. A hoisted developer message keeps only role, content, name, and cache_control, a closing developer message's content is deep-copied so the caller's list is left alone, and a hosted_vllm developer item with no content is forwarded unchanged. The Fireworks in-place test now declares true on the deployment itself, because a cost-map true can no longer be told apart from another deployment's registration. --------- Co-authored-by: sowmy narayanan Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/anthropic/chat/transformation.py | 5 +- .../messages/native/transformation.py | 5 +- litellm/llms/azure/chat/gpt_transformation.py | 19 +- litellm/llms/base_llm/base_utils.py | 238 ++++++++- litellm/llms/base_llm/chat/transformation.py | 15 +- .../fireworks_ai/responses/transformation.py | 128 +---- .../hosted_vllm/responses/transformation.py | 28 + .../llms/openai/chat/gpt_transformation.py | 75 ++- .../openai/chat/o_series_transformation.py | 12 +- litellm/llms/openai/openai.py | 21 +- .../openai/responses/instruction_items.py | 98 ++++ litellm/main.py | 17 +- ...odel_prices_and_context_window_backup.json | 2 + litellm/utils.py | 37 ++ model_prices_and_context_window.json | 2 + tests/unit/conftest.py | 14 + .../test_azure_chat_gpt_transformation.py | 23 + .../chat/test_azure_ai_transformation.py | 23 + tests/unit/llms/base_llm/test_base_utils.py | 494 ++++++++++++++++++ .../test_fireworks_ai_chat_transformation.py | 84 +++ ...t_fireworks_ai_responses_transformation.py | 102 +++- .../responses/test_hosted_vllm_responses.py | 93 ++++ .../test_litellm_proxy_chat_transformation.py | 22 + .../chat/test_openai_gpt_transformation.py | 96 ++++ .../openai/test_o_series_transformation.py | 17 + tests/unit/llms/openai/test_openai.py | 40 +- tests/unit/test_utils.py | 38 ++ 28 files changed, 1612 insertions(+), 137 deletions(-) create mode 100644 litellm/llms/openai/responses/instruction_items.py create mode 100644 tests/unit/llms/base_llm/test_base_utils.py diff --git a/litellm/constants.py b/litellm/constants.py index 7f2d98b7a28..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 6c0e463bd23..75b0329315f 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -207,7 +208,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -218,7 +219,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/main.py b/litellm/main.py index 996e93589fe..a3a63d5a83a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5703,15 +5703,16 @@ def completion( ), ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5808,7 +5809,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50b3560f05b..aa00ea73f8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/litellm/utils.py b/litellm/utils.py index aeb25f81841..adf195668ee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -3501,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50b3560f05b..aa00ea73f8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3070ebb0235..ed7dd5ec146 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -378,6 +378,20 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +@pytest.fixture +def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable[[str, bool], None]: + def declare(model_cost_key: str, supported: bool) -> None: + entry: Final = { + "litellm_provider": model_cost_key.split("/", 1)[0], + "mode": "chat", + "supports_mid_conversation_system": supported, + } + monkeypatch.setitem(litellm.model_cost, model_cost_key, entry) + litellm.get_model_info.cache_clear() + + return declare + + NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 53df6610913..f6dae2c304f 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -783,3 +783,26 @@ def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mo assert request.url.params["api-version"] == "2024-12-01-preview" assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} assert response.choices[0].message.content == "Hello" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("config", [AzureOpenAIConfig(), AzureOpenAIGPT5Config()]) +def test_azure_keeps_a_later_developer_message_in_place_as_system(config: AzureOpenAIConfig) -> None: + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="my-deployment", + custom_llm_provider="azure", + api_base="https://example-resource.openai.azure.com", + model_info=None, + ) + assert [message["role"] for message in messages] == ["system", "user", "assistant", "system", "user"] + assert messages[3]["content"] == "Answer with exactly one word." diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index ccac6ffc309..42dc222e420 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,5 +1,6 @@ import json import traceback +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -1098,3 +1099,25 @@ def test_map_azure_model_group(model_group_header, expected_model): config = AzureAICohereConfig() assert config._map_azure_model_group(model_group_header) == expected_model + + +def test_translate_developer_role_hoists_a_later_developer_message_for_a_model_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("azure_ai/system-first-model", False) + messages: Final = AzureAIStudioConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="system-first-model", + custom_llm_provider="azure_ai", + api_base="https://example.services.ai.azure.com/models", + model_info=None, + ) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user"] + assert messages[0]["content"] == "You are terse.\n\nAnswer with exactly one word." diff --git a/tests/unit/llms/base_llm/test_base_utils.py b/tests/unit/llms/base_llm/test_base_utils.py new file mode 100644 index 00000000000..7052fd04242 --- /dev/null +++ b/tests/unit/llms/base_llm/test_base_utils.py @@ -0,0 +1,494 @@ +from typing import Final + +from litellm.llms.base_llm.base_utils import ( + hoist_developer_messages_into_leading_system_message, + merge_system_message_runs, +) + + +class TestMergeSystemMessageRuns: + def test_later_developer_message_stays_in_place_while_the_leading_run_folds(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Sandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex\n\nSandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_run_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + + def test_hoisted_developer_message_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": 42}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": 42}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +class TestHoistDeveloperMessagesIntoLeadingSystemMessage: + def test_single_developer_message(self) -> None: + messages: Final = [ + {"role": "developer", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_developer_messages_merge(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Rule 1: Be concise."}, + {"role": "developer", "content": "Rule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rule 1: Be concise.\n\nRule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + + def test_system_followed_by_developer_merges(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base instructions."}, + {"role": "developer", "content": "Developer override."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base instructions.\n\nDeveloper override."}, + {"role": "user", "content": "Hello"}, + ] + + def test_developer_followed_by_system_merges(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Developer instruction."}, + {"role": "system", "content": "System instruction."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Developer instruction.\n\nSystem instruction."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_native_system_messages_merge(self) -> None: + messages: Final = [ + {"role": "system", "content": "System part 1."}, + {"role": "system", "content": "System part 2."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System part 1.\n\nSystem part 2."}, + {"role": "user", "content": "Hello"}, + ] + + def test_empty_string_side_does_not_pad(self) -> None: + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": ""}, {"role": "developer", "content": "Rules"}] + ) + ) == [{"role": "system", "content": "Rules"}] + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": "Rules"}, {"role": "developer", "content": ""}] + ) + ) == [{"role": "system", "content": "Rules"}] + + def test_empty_string_message_adds_no_block_when_the_run_merges_as_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": ""}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + + def test_two_cached_string_messages_keep_one_breakpoint_each(self) -> None: + messages: Final = [ + {"role": "system", "content": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"role": "system", "content": "B", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + }, + ] + + def test_cache_control_supplied_only_by_the_later_message_marks_only_its_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "A"}, + {"role": "developer", "content": "B", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A"}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + def test_cache_control_of_the_first_message_stays_on_its_own_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "developer", "content": "Additional instructions"}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAdditional instructions"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_cached_string_message_merged_with_block_content_keeps_its_breakpoint(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": [{"type": "text", "text": "Answer with exactly one word."}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_plain_messages_ahead_of_a_cached_one_keep_their_own_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "developer", "content": "Rule"}, + {"role": "developer", "content": "Cached", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Base"}, + {"type": "text", "text": "\n\nRule"}, + {"type": "text", "text": "\n\nCached", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_name_of_the_last_named_message_survives_a_long_run(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base", "name": "first"}, + *({"role": "developer", "content": f"Rule {index}"} for index in range(2000)), + {"role": "developer", "content": "Last", "name": "last"}, + {"role": "user", "content": "Hello"}, + ] + merged: Final = list(hoist_developer_messages_into_leading_system_message(messages)) + assert len(merged) == 2 + assert merged[0]["name"] == "last" + assert merged[0]["content"] == "\n\n".join(["Base", *(f"Rule {index}" for index in range(2000)), "Last"]) + + def test_billing_metadata_keeps_its_own_block_when_merged(self) -> None: + billing: Final = {"role": "system", "content": "x-anthropic-billing-header: cc_version=1.0;"} + advisory: Final = {"role": "system", "content": "Guardrail advisory: request was flagged"} + assert list(hoist_developer_messages_into_leading_system_message([billing, advisory])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + {"type": "text", "text": "\n\nGuardrail advisory: request was flagged"}, + ], + } + ] + assert list(hoist_developer_messages_into_leading_system_message([advisory, billing])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Guardrail advisory: request was flagged"}, + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + ], + } + ] + + def test_developer_message_after_a_user_turn_is_hoisted_into_the_leading_system_message(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "developer", "content": "Developer turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System turn 1\n\nDeveloper turn 2"}, + {"role": "user", "content": "User turn 1"}, + {"role": "user", "content": "User turn 2"}, + ] + + def test_hoisted_developer_message_keeps_only_the_fields_a_system_message_has(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word.", "name": "codex", "extra_key": "x"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word.", "name": "codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_developer_message_in_second_position_without_a_leading_system_message_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_every_later_developer_message_is_hoisted_in_order_across_assistant_and_tool_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "developer", "content": "Update B"}, + {"role": "user", "content": "Turn 3"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A\n\nUpdate B"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "user", "content": "Turn 3"}, + ] + + def test_developer_message_that_closes_the_conversation_after_an_assistant_turn_becomes_a_user_turn(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with the single word PONG and nothing else."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "user", "content": "Reply with the single word PONG and nothing else."}, + ] + + def test_closing_developer_message_keeps_its_cache_mark_as_a_user_turn(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with PONG.", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages))[-1] == { + "role": "user", + "content": "Reply with PONG.", + "cache_control": {"type": "ephemeral"}, + } + + def test_only_the_closing_developer_run_after_an_assistant_turn_becomes_user_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "developer", "content": "Closing B"}, + {"role": "developer", "content": "Closing C"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A"}, + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "user", "content": "Closing B"}, + {"role": "user", "content": "Closing C"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_user_turn_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Reply with exactly one word: the capital of France"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Reply with exactly one word: the capital of France"}, + {"role": "user", "content": "Hi there"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_tool_result_is_hoisted(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + {"role": "developer", "content": "Answer with exactly one word."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + ] + + def test_hoisted_single_text_block_developer_message_joins_string_instructions_as_text(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "developer", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex\n\n"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + + def test_multi_block_developer_message_keeps_its_blocks_after_string_instructions(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi"}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Codex"}, + {"type": "text", "text": "\n\nRule 1"}, + {"type": "text", "text": "Rule 2"}, + ], + }, + {"role": "user", "content": "Hi"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_system_message_without_content_folds_into_the_run_instead_of_raising(self) -> None: + messages: Final = [ + {"role": "system", "content": "Rules"}, + {"role": "system"}, + {"role": "system", "content": None}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1"}, + {"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "system", "content": "System turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == messages + + def test_list_content_merge_preserves_block_level_cache_control(self) -> None: + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}]}, + {"role": "developer", "content": [{"type": "text", "text": "Extra"}]}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nExtra"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 61e5545b8bf..28f4fdead9e 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -2358,3 +2359,86 @@ def test_global_disable_flag_with_transform_messages_helper( ) assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] + + +_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +def test_translate_developer_role_keeps_a_later_developer_message_in_place_for_an_undeclared_fireworks_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +def test_translate_developer_role_leaves_a_leading_system_and_developer_pair_unmerged_for_an_undeclared_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("model", ["qwen3p8-max", "accounts/fireworks/models/qwen3p8-max"]) +def test_translate_developer_role_hoists_a_later_developer_message_for_the_system_first_qwen3p8_model( + local_model_cost_map: None, model: str +) -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model=model, + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("deployment_declared", "expected_roles"), + [(False, ["system", "user", "user"]), (True, ["system", "user", "system", "user"])], +) +def test_translate_developer_role_follows_the_deployment_model_info_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], deployment_declared: bool, expected_roles: list[str] +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", not deployment_declared) + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/shared-backend", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info={"id": "deployment-a", "supports_mid_conversation_system": deployment_declared}, + ) + + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 878e0285cd1..7fc6ac2c436 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -1,5 +1,5 @@ import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final, TypedDict, cast from unittest.mock import MagicMock, patch @@ -7,6 +7,7 @@ from urllib.parse import quote import httpx import pytest +import respx from openai.types.responses import ( EasyInputMessage, ResponseFunctionToolCall, @@ -81,6 +82,14 @@ def _mock_http_client(response_body: Mapping[str, object]) -> MagicMock: return client +def _fireworks_responses_route(model: str) -> respx.Route: + return respx.post(FIREWORKS_RESPONSES_URL).mock(return_value=httpx.Response(200, json=_fireworks_response(model))) + + +def _routed_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + def _sent_request(client: MagicMock) -> tuple[str, Mapping[str, str], Mapping[str, object]]: kwargs: Final = client.post.call_args.kwargs body: Final = kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) @@ -182,6 +191,60 @@ def test_responses_call_folds_developer_items_into_instructions() -> None: ) +@respx.mock +def test_responses_call_keeps_developer_items_in_place_for_a_deployment_that_accepts_later_system_items() -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/accepts-later-system") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/accepts-later-system", + instructions="You are Codex", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model_info={"id": "deployment-1", "supports_mid_conversation_system": True}, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "You are Codex" + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, + {"role": "user", "content": "What is the capital of France?"}, + ) + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "closing_role"), + [(None, "system"), ({"id": "deployment-1", "supports_mid_conversation_system": False}, "user")], +) +def test_responses_call_sends_the_closing_developer_item_as_a_user_turn_only_for_a_declared_system_first_deployment( + model_info: Mapping[str, object] | None, closing_role: str +) -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/kimi-k3") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Reply with PONG."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "Answer with exactly one word." + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": closing_role, "content": "Reply with PONG.", "type": "message"}, + ) + + def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): @@ -667,3 +730,40 @@ def test_responses_call_forwards_litellm_user_id_as_user( _, _, body = _sent_request(client) assert body.get("user") == expected_user assert "fireworks_forward_user_id" not in body + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "expected_instructions", "expected_input"), + [ + (None, "Answer in one word.", ({"role": "user", "content": "What is the capital of France?"},)), + ( + {"id": "deployment-1", "supports_mid_conversation_system": True}, + None, + ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Answer in one word.", "type": "message"}, + ), + ), + ], +) +def test_responses_call_forwards_developer_items_in_place_only_when_the_deployment_itself_declares_true( + declare_mid_conversation_system: Callable[[str, bool], None], + model_info: Mapping[str, object] | None, + expected_instructions: str | None, + expected_input: tuple[Mapping[str, object], ...], +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", True) + route: Final = _fireworks_responses_route("accounts/fireworks/models/shared-backend") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/shared-backend", + input=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "developer", "content": "Answer in one word."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body.get("instructions") == expected_instructions + assert tuple(body["input"]) == expected_input diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..2f63ef070e6 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,9 +8,13 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm @@ -183,3 +187,92 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +_LATER_DEVELOPER_INPUT: Final = ( + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +_VLLM_RESPONSES_URL: Final = "https://test-vllm.example.com/v1/responses" + + +def _vllm_responses_route(output_text: str) -> respx.Route: + return respx.post(_VLLM_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_make_mock_responses_api_response(output_text)) + ) + + +def _sent_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + +@respx.mock +def test_hosted_vllm_responses_folds_a_later_developer_item_into_instructions_for_a_system_first_model( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("hosted_vllm/Qwen/Qwen3.6-27B", False) + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3.6-27B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == (_LATER_DEVELOPER_INPUT[0], _LATER_DEVELOPER_INPUT[2]) + + +@respx.mock +def test_hosted_vllm_responses_folds_for_a_deployment_declared_system_first_and_sends_a_closing_developer_as_user() -> None: + assistant_turn: Final = {"type": "message", "role": "assistant", "content": "Paris"} + route: Final = _vllm_responses_route("PONG") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=[*_LATER_DEVELOPER_INPUT, assistant_turn, {"role": "developer", "content": "Reply with PONG."}], + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == ( + _LATER_DEVELOPER_INPUT[0], + _LATER_DEVELOPER_INPUT[2], + assistant_turn, + {"role": "user", "content": "Reply with PONG.", "type": "message"}, + ) + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_later_developer_item_in_place_for_an_undeclared_model() -> None: + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant." + assert tuple(body["input"]) == _LATER_DEVELOPER_INPUT + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_developer_item_without_content_on_a_deployment_declared_system_first() -> None: + route: Final = _vllm_responses_route("Paris") + input_items: Final = ( + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"type": "message", "role": "developer"}, + ) + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input=list(input_items), + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + assert tuple(_sent_body(route)["input"]) == input_items diff --git a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 9b8788d3769..8a690a70a97 100644 --- a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -639,3 +639,25 @@ def test_litellm_gateway_from_sdk_with_thinking_param(respx_mock): ) e = exc_info.value assert "Connection error." in str(e) + + +def test_translate_developer_role_leaves_a_later_developer_message_in_place_for_the_downstream_proxy() -> None: + translated: Final = LiteLLMProxyChatConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="fireworks-qwen", + custom_llm_provider="litellm_proxy", + api_base="http://inner-proxy:4000", + model_info=None, + ) + + assert list(translated) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 70436662ef0..5b85a94b24d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,7 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ import base64 +import copy import json +from collections.abc import Callable from typing import Final import pytest @@ -1820,3 +1822,97 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: ) assert request["tools"][0] is tool + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("config", "custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + (OpenAIGPTConfig(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPT5Config(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", False, ["system", "user", "assistant", "user"]), + (OpenAIGPTConfig(), "groq", None, None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "groq", None, False, ["system", "user", "assistant", "user"]), + ], +) +def test_translate_developer_role_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + config: OpenAIGPTConfig, + custom_llm_provider: str, + api_base: str | None, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles + assert "Answer with exactly one word." in [message["content"] for message in messages if message["role"] == "system"][-1] + + +@pytest.mark.parametrize("role", [["user"], {"name": "user"}]) +@pytest.mark.parametrize( + ("declared", "expected_order"), + [(None, ("caller", "instruction")), (False, ("instruction", "caller"))], +) +def test_translate_developer_role_places_a_message_with_a_non_string_role_like_any_conversation_turn( + declare_mid_conversation_system: Callable[[str, bool], None], + role: object, + declared: bool | None, + expected_order: tuple[str, str], +) -> None: + if declared is not None: + declare_mid_conversation_system("groq/system-first-model", declared) + caller_turn: Final = {"role": role, "content": "Hi there"} + by_name: Final = {"caller": caller_turn, "instruction": {"role": "system", "content": "Answer in one word."}} + messages: Final = OpenAIGPTConfig().translate_developer_role_to_system_role( + messages=[caller_turn, {"role": "developer", "content": "Answer in one word."}], + model="system-first-model", + custom_llm_provider="groq", + api_base=None, + model_info=None, + ) + assert list(messages) == [by_name[name] for name in expected_order] + + +@pytest.mark.respx(assert_all_called=True) +def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_system_first_deployment( + respx_mock: respx.MockRouter, +) -> None: + chat: Final = respx_mock.post("http://vllm.internal:8000/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + messages: Final = [ + {"role": "user", "content": "Describe the image."}, + {"role": "assistant", "content": "Send it."}, + { + "role": "developer", + "content": [{"type": "text", "text": "Look at this."}, {"type": "image_url", "image_url": "data:image/png;base64,aW1hZ2U="}], + }, + ] + sent: Final = copy.deepcopy(messages) + litellm.completion( + model="openai/system-first-model", + api_base="http://vllm.internal:8000/v1", + api_key="sk-test", + messages=messages, + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + ) + assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" + assert messages == sent diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index 121fab66c15..e78e1a6f12b 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -181,3 +181,20 @@ def test_litellm_responses(): print("response: ", response) assert isinstance(response.usage.completion_tokens_details, CompletionTokensDetails) + + +def test_translate_developer_role_keeps_developer_messages_untouched() -> None: + messages: Final = [ + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert ( + OpenAIOSeriesConfig().translate_developer_role_to_system_role( + messages=messages, + model="my-reasoning-deployment", + custom_llm_provider="openai", + api_base=None, + model_info=None, + ) + == messages + ) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 11d7c7d6d3c..5b27a446fb3 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2,7 +2,7 @@ import asyncio import importlib import json import os -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncCursorPage, MessageData, OpenAIChatCompletion, + OpenAIConfig, OpenAIMessage as Message, Run, SyncCursorPage, @@ -2010,3 +2011,40 @@ async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_head assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} assert sent.headers["x-caller"] == "unit" assert sent.headers["authorization"] == "Bearer sk-test" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + ("openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", None, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", False, ["system", "user", "assistant", "user"]), + ], +) +def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + custom_llm_provider: str, + api_base: str, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = OpenAIConfig().translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..2870016a5e9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10869,6 +10869,44 @@ def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_e assert "cache_control_injection_points" not in optional_params +@pytest.mark.parametrize( + ("cost_map_declared", "deployment_declared", "expected"), + [ + (None, None, None), + (False, None, False), + (True, None, True), + (True, False, False), + (False, True, True), + (None, False, False), + ], +) +def test_declared_mid_conversation_system_support_prefers_the_deployment_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], + cost_map_declared: bool | None, + deployment_declared: bool | None, + expected: bool | None, +) -> None: + if cost_map_declared is not None: + declare_mid_conversation_system("hosted_vllm/mid-conversation-probe", cost_map_declared) + deployment_model_info: Final = ( + {"id": "deployment-1"} + if deployment_declared is None + else {"id": "deployment-1", "supports_mid_conversation_system": deployment_declared} + ) + assert ( + litellm.utils.declared_mid_conversation_system_support( + "mid-conversation-probe", "hosted_vllm", deployment_model_info + ) + is expected + ) + + +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") + litellm.get_model_info.cache_clear() + assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None + + def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy applies cache_control_injection_points itself, so the point must ride along in extra_body.""" From 0cf2bb7296cf2c1546ba29ceb2236d4d9408ae94 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:46:29 -0700 Subject: [PATCH 2/3] fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params (#47761) * fix(openai): warn when the GPT-5 reasoning effort gate drops a level under drop_params A GPT-5 request whose reasoning_effort the model map does not enable for that model lost the effort with no log line under drop_params. An Azure deployment name missing from the cost map lost reasoning_effort none this way and ran at the model's default effort. The gate now logs a warning naming the model, the level, the map entry it checked, and the model_info.base_model and model_info.reasoning_effort_levels fixes. * fix(openai): warn when the Responses route or a deployment's declared levels drop a reasoning effort The /v1/responses gate dropped a row-disabled GPT-5 effort, and reasoning for an OpenAI model without supports_reasoning, with no log line, and the router logged a declared-level drop only at INFO. Both Responses drops now log a WARNING and the router logs its drop at WARNING. The warning now leads with model_info.reasoning_effort_levels, since base_model only helps a deployment named differently from its underlying model. * fix: warn only when a reasoning level is actually dropped, and on generic reasoning_effort drops * fix: warn when thinking or a Bedrock Converse reasoning level is dropped, and name the router drop a reasoning level * fix: skip the drop warning when a non-reasoning model is asked for no reasoning, and honor declared levels on Bedrock Responses * fix: stay silent when reasoning is turned off on /v1/responses and warn on the Bedrock Responses fallback drop * fix(reasoning): warn when any reasoning-off request is dropped and give each route only the advice it honors A request to turn reasoning off on a model name the catalog lacks now warns on every route, since it may be a reasoning model. Chat suggests model_info.reasoning_effort_levels and, for a name with no catalog entry, model_info.base_model. OpenAI Responses suggests declared levels only where the config honors them, and Bedrock suggests neither. The Bedrock and Mantle Responses forwarding of declared levels is reverted until it can be proven live. * fix(reasoning): suggest reasoning_effort_levels and base_model only where the route reads them The chat gate suggested model_info.reasoning_effort_levels on routes that never pass declared levels (github_copilot, chatgpt and other OpenAIConfig delegates) and model_info.base_model on OpenAI and Azure AI chat, whose gate looks up the deployment name rather than base_model. /v1/responses/compact suggested levels it never reads either. Declared levels are now None when a caller passes none, so the advice appears only when a caller fed them, and the base_model advice is limited to the Azure chat gate, the one that looks up base_model. The openai row of the chat warning test now expects no base_model advice, the Responses warning test goes through get_optional_params_responses_api the way /v1/responses and compact call it, and the Azure direct-config test passes declared levels the way get_optional_params does. Long test lines are wrapped. * fix: read only the built-in catalog when choosing the reasoning drop warning --- .../chat/gpt_reasoning_transformation.py | 3 +- litellm/llms/azure_ai/chat/transformation.py | 2 +- .../llms/azure_ai/responses/transformation.py | 2 +- .../bedrock/chat/converse_transformation.py | 7 +- litellm/llms/bedrock/common_utils.py | 6 + .../llms/bedrock/responses/transformation.py | 3 +- .../chat/gpt_reasoning_transformation.py | 37 ++++- .../llms/openai/responses/transformation.py | 41 ++++- litellm/responses/utils.py | 8 +- .../reasoning_effort_capability.py | 30 +++- litellm/utils.py | 93 +++++++++-- ...test_azure_gpt_reasoning_transformation.py | 24 +++ .../chat/test_converse_transformation.py | 21 ++- .../test_bedrock_openai_responses.py | 42 ++++- ...bedrock_mantle_responses_transformation.py | 27 ++++ .../test_openai_responses_transformation.py | 101 ++++++++++++ .../test_gpt_reasoning_transformation.py | 53 +++++++ .../test_reasoning_effort_capability.py | 26 +++- tests/unit/test_utils_get_optional_params.py | 144 ++++++++++++++++++ 19 files changed, 609 insertions(+), 61 deletions(-) diff --git a/litellm/llms/azure/chat/gpt_reasoning_transformation.py b/litellm/llms/azure/chat/gpt_reasoning_transformation.py index 5bd03f511c5..72a0500e0af 100644 --- a/litellm/llms/azure/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/azure/chat/gpt_reasoning_transformation.py @@ -24,6 +24,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): GPT5_SERIES_ROUTE = "gpt5_series/" OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none")) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = True @classmethod def _model_map_lookup_name(cls, model: str) -> str: @@ -88,7 +89,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config): drop_params: bool, api_version: str = "", *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: return OpenAIGPT5Config.map_openai_params( self, diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 6aed7eb3c54..8bb6072d0a9 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -106,7 +106,7 @@ class AzureAIStudioConfig(OpenAIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict[str, object]: # mutable-ok: OpenAIConfig.map_openai_params signature if not azureAIGPT5Config.is_model_gpt_5_model(model): return super().map_openai_params( diff --git a/litellm/llms/azure_ai/responses/transformation.py b/litellm/llms/azure_ai/responses/transformation.py index da734cf8189..b19915648ab 100644 --- a/litellm/llms/azure_ai/responses/transformation.py +++ b/litellm/llms/azure_ai/responses/transformation.py @@ -232,7 +232,7 @@ class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: mapped: Final = super().map_openai_params_for_deployment( response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index f54c352fd6e..05440d70017 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -117,6 +117,7 @@ from litellm.utils import ( ) from ..common_utils import ( + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, BedrockModelInfo, bedrock_cache_point_ttl_supported, @@ -1192,11 +1193,7 @@ class AmazonConverseConfig(BaseConfig): ), status_code=400, ) - verbose_logger.debug( - "Dropping unsupported `reasoning_effort=%s` for Bedrock model=%s.", - value, - model, - ) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning_effort", value, model) elif param == "reasoning_effort" and isinstance(value, str): self._handle_reasoning_effort_parameter( model=model, reasoning_effort=value, optional_params=optional_params diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index dd721e059a9..4175479f068 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -1331,6 +1331,12 @@ def _mantle_api_base_from_env() -> str | None: return next((base[: -len(suffix)] for suffix in _MANTLE_OPENAI_BASE_SUFFIXES if base.endswith(suffix)), base) +DROP_DISABLED_REASONING_EFFORT_WARNING: Final = ( + "Dropping %s=%r for Bedrock model=%r (drop_params=True) because the model map marks that level unsupported. " + "Send a level the model supports." +) + + def bedrock_reasoning_effort_disabled(model: str, effort: str) -> bool: from litellm.utils import is_explicitly_disabled_factory diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 7cdd7b08f99..ff44d973d97 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -52,6 +52,7 @@ from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import ( BEDROCK_CHAT_COMPLETIONS_ROUTE_PREFIX, + DROP_DISABLED_REASONING_EFFORT_WARNING, BedrockError, bedrock_reasoning_effort_disabled, bedrock_supports_openai_responses, @@ -173,7 +174,7 @@ def _without_disabled_reasoning_effort( ), status_code=400, ) - verbose_logger.debug("Dropping unsupported `reasoning.effort=%s` for Bedrock model=%s.", effort, model) + verbose_logger.warning(DROP_DISABLED_REASONING_EFFORT_WARNING, "reasoning.effort", effort, model) rest: Final = {key: value for key, value in reasoning.items() if key != "effort"} without_reasoning: Final = {key: value for key, value in params.items() if key != "reasoning"} return {**without_reasoning, "reasoning": rest} if rest else without_reasoning diff --git a/litellm/llms/openai/chat/gpt_reasoning_transformation.py b/litellm/llms/openai/chat/gpt_reasoning_transformation.py index 0311f6c6590..8c9551d01c0 100644 --- a/litellm/llms/openai/chat/gpt_reasoning_transformation.py +++ b/litellm/llms/openai/chat/gpt_reasoning_transformation.py @@ -5,8 +5,10 @@ from collections.abc import Collection from typing import ClassVar, Final import litellm +from litellm._logging import verbose_logger from litellm.utils import ( declared_value_factory, + has_catalog_entry, is_explicitly_disabled_factory, supports_factory, ) @@ -16,6 +18,25 @@ from .gpt_transformation import OpenAIGPTConfig _GPT_SERIES_VERSION: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?(?=[.-]|$)") +def refused_reasoning_effort_warning(*, suggest_declared_levels: bool, suggest_base_model: bool) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because the model map does not mark that level " + "supported for %r.", + ( + " If the deployment supports it, list every level it supports in model_info.reasoning_effort_levels." + if suggest_declared_levels + else "" + ), + ( + " If the deployment name is not its underlying model, set model_info.base_model to that model." + if suggest_base_model + else "" + ), + ) + ) + + def _catalogue_declares_default_effort() -> bool: """Whether the loaded cost map carries default_reasoning_effort for ANY entry. @@ -93,6 +114,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",)) ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",)) + LOOKS_UP_BASE_MODEL: ClassVar[bool] = False @classmethod def is_model_gpt_5_model(cls, model: str) -> bool: @@ -307,7 +329,7 @@ class OpenAIGPT5Config(OpenAIGPTConfig): model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: if self.is_model_gpt_5_search_model(model): if "max_tokens" in non_default_params: @@ -338,10 +360,21 @@ class OpenAIGPT5Config(OpenAIGPTConfig): if ( requested_effort is not None - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self.refuses_reasoning_effort_level(model, requested_effort) ): if litellm.drop_params or drop_params: + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, + suggest_base_model=self.LOOKS_UP_BASE_MODEL + and not has_catalog_entry(self._model_map_lookup_name(model), None), + ), + "reasoning_effort", + requested_effort, + model, + self._model_map_lookup_name(model), + ) non_default_params.pop("reasoning_effort", None) optional_params.pop("reasoning_effort", None) else: diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 903e6177e7b..9b8515a9ecd 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -32,6 +32,7 @@ from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_a from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_reasoning_transformation import ( is_gpt_reasoning_series_name, + refused_reasoning_effort_warning, unsupported_reasoning_effort_message, ) from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE @@ -55,6 +56,9 @@ from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 +DROP_UNSUPPORTED_REASONING_WARNING: Final = ( + "Dropping reasoning=%r for model=%r (drop_params=True) because its model cost map entry lacks supports_reasoning." +) _RAW_RESPONSE_JSON: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True)) if TYPE_CHECKING: @@ -417,12 +421,19 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return declared return info["key"] in _bundled_openai_reasoning_models() + @staticmethod + def _is_known_non_reasoning_model(model: str) -> bool: + from litellm.utils import is_known_non_reasoning_model + + return is_known_non_reasoning_model(model.split("/")[-1], LlmProviders.OPENAI.value) + @staticmethod def _requests_reasoning_effort(reasoning: object) -> bool: - effort: Final = ( - reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) - ) - return effort is not None + return OpenAIResponsesAPIConfig._requested_effort(reasoning) is not None + + @staticmethod + def _requested_effort(reasoning: object) -> object: + return reasoning.get("effort") if isinstance(reasoning, Mapping) else getattr(reasoning, "effort", None) @staticmethod def _enforce_min_max_output_tokens(max_output_tokens: "int | None") -> "int | None": @@ -464,17 +475,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): ) -> dict: return dict( self.map_openai_params_for_deployment( - response_api_optional_params, model, drop_params, declared_reasoning_efforts=() + response_api_optional_params, model, drop_params, declared_reasoning_efforts=None ) ) + def honors_declared_reasoning_efforts(self) -> bool: + return type(self).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + def map_openai_params_for_deployment( self, response_api_optional_params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, *, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: """No mapping applied since inputs are in OpenAI spec already. @@ -494,7 +508,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): and not self._supports_reasoning_param(model=model) ): if drop_params or litellm.drop_params: - params.pop("reasoning", None) + dropped_reasoning: Final = params.pop("reasoning", None) + if self._requested_effort(dropped_reasoning) != "none" or not self._is_known_non_reasoning_model(model): + verbose_logger.warning(DROP_UNSUPPORTED_REASONING_WARNING, dropped_reasoning, model) else: raise litellm.UnsupportedParamsError( message=( @@ -512,7 +528,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): if ( isinstance(reasoning, dict) and isinstance(requested_effort, str) - and requested_effort not in declared_reasoning_efforts + and requested_effort not in (declared_reasoning_efforts or ()) and self._effort_level_is_disabled(lookup_name, requested_effort) ): if not (drop_params or litellm.drop_params): @@ -520,6 +536,15 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model), status_code=400, ) + verbose_logger.warning( + refused_reasoning_effort_warning( + suggest_declared_levels=declared_reasoning_efforts is not None, suggest_base_model=False + ), + "reasoning.effort", + requested_effort, + model, + lookup_name, + ) remaining: Final = reasoning.copy() remaining.pop("effort", None) if remaining: diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 05d6a36c1f6..b0db0516658 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -131,14 +131,14 @@ def _provider_mapped_params( params: ResponsesAPIOptionalRequestParams, model: str, drop_params: bool, - declared_reasoning_efforts: Collection[str], + declared_reasoning_efforts: Collection[str] | None, ) -> Mapping[str, object]: from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig if ( - declared_reasoning_efforts + declared_reasoning_efforts is not None and isinstance(config, OpenAIResponsesAPIConfig) - and type(config).map_openai_params is OpenAIResponsesAPIConfig.map_openai_params + and config.honors_declared_reasoning_efforts() ): return config.map_openai_params_for_deployment( params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts @@ -287,7 +287,7 @@ class ResponsesAPIRequestUtils: response_api_optional_params: ResponsesAPIOptionalRequestParams, allowed_openai_params: list[str] | None = None, drop_params: bool | None = None, - declared_reasoning_efforts: Collection[str] = (), + declared_reasoning_efforts: Collection[str] | None = None, ) -> dict: """ Get optional parameters for the responses API. diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py index 16b8a22a40f..3c3fad93579 100644 --- a/litellm/router_utils/reasoning_effort_capability.py +++ b/litellm/router_utils/reasoning_effort_capability.py @@ -426,6 +426,27 @@ def _log_remap( logged_deployment_id: Final = _single_line(deployment_id) logged_model_group: Final = _single_line(model_group) logged_deployment_model: Final = _single_line(deployment_model) + extra: Final = MappingProxyType( + { + "reasoning_effort_deployment_id": logged_deployment_id, + "reasoning_effort_model_group": logged_model_group, + "reasoning_effort_deployment_model": logged_deployment_model, + "reasoning_effort_requested": logged_requested_level, + "reasoning_effort_sent": logged_sent_level, + "reasoning_effort_rounding": rounding, + } + ) + if sent_level is None and requested_level != _DEFAULT_EFFORT: + verbose_logger.warning( + "Dropping reasoning level %s (drop_params=True) for deployment %s, model group %s, model %s because its " + "model_info.reasoning_effort_levels does not list that level. List every level the deployment supports.", + logged_requested_level, + logged_deployment_id, + logged_model_group, + logged_deployment_model, + extra=extra, + ) + return verbose_logger.info( "reasoning_effort %s sent as %s (rounding %s) to deployment %s, model group %s, model %s", logged_requested_level, @@ -434,14 +455,7 @@ def _log_remap( logged_deployment_id, logged_model_group, logged_deployment_model, - extra={ - "reasoning_effort_deployment_id": logged_deployment_id, - "reasoning_effort_model_group": logged_model_group, - "reasoning_effort_deployment_model": logged_deployment_model, - "reasoning_effort_requested": logged_requested_level, - "reasoning_effort_sent": logged_sent_level, - "reasoning_effort_rounding": rounding, - }, + extra=extra, ) diff --git a/litellm/utils.py b/litellm/utils.py index adf195668ee..a656d8ebca3 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -91,7 +91,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( match_capability_generalizations, match_fill_missing_generalizations, ) -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry, GetModelCostMap from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload @@ -3852,7 +3852,7 @@ def _resolve_builtin_model_cost_entry(key: str, provider: str) -> _BuiltinCostMa return None -def is_generalized_model_info(model_info: ModelInfo) -> bool: +def is_generalized_model_info(model_info: ModelInfoBase) -> bool: """Whether ``model_info`` came from a fallback-generalization capability rule. Detected as the resolved key missing ``litellm.model_cost`` while matching a @@ -3868,10 +3868,12 @@ def is_generalized_model_info(model_info: ModelInfo) -> bool: _REGISTRATION_ATTEMPTS: Final = 3 +_ModelInfoT: Final = TypeVar("_ModelInfoT", bound=ModelInfoBase) -def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + +def _lookup_mapped_model_info(lookup: Callable[[], _ModelInfoT]) -> _ModelInfoT | ModelInfoLookupMiss: try: - info: Final = get_model_info(model=model, custom_llm_provider=custom_llm_provider) + info: Final = lookup() except ModelNotMappedError: return ModelInfoLookupMiss(tripped=False) except Exception: @@ -3879,6 +3881,16 @@ def lookup_model_info_for_registration(model: str, custom_llm_provider: str | No return ModelInfoLookupMiss(tripped=False) if is_generalized_model_info(info) else info +def lookup_model_info_for_registration(model: str, custom_llm_provider: str | None) -> ModelInfo | ModelInfoLookupMiss: + return _lookup_mapped_model_info(lambda: get_model_info(model=model, custom_llm_provider=custom_llm_provider)) + + +def lookup_catalog_model_info(model: str, custom_llm_provider: str | None) -> ModelInfoBase | ModelInfoLookupMiss: + return _lookup_mapped_model_info( + lambda: get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider, catalog_only=True) + ) + + def _get_builtin_model_info_for_registration( model: str, custom_llm_provider: str | None ) -> ModelInfo | ModelInfoLookupMiss: @@ -5144,6 +5156,46 @@ def _declared_reasoning_efforts(model_info: Mapping[str, object] | None) -> tupl return deployment_reasoning_effort_levels(model_info) or () +_DROPPED_REASONING_PARAMS: Final = ("reasoning_effort", "thinking") + + +def _turns_reasoning_off(value: object) -> bool: + match value: + case "none" | {"effort": "none"} | {"type": "disabled"}: + return True + case _: + return False + + +def is_known_non_reasoning_model(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + if isinstance(info, ModelInfoLookupMiss) or info.get("supports_reasoning") is True: + return False + return info.get("supports_reasoning") is False or info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def has_catalog_entry(model: str, custom_llm_provider: str | None) -> bool: + info: Final = lookup_catalog_model_info(model, custom_llm_provider) + return not isinstance(info, ModelInfoLookupMiss) and info["key"] in GetModelCostMap.loaded_model_cost_map() + + +def _dropped_reasoning_warning(model: str, custom_llm_provider: str | None) -> str: + return "".join( + ( + "Dropping %s=%r for model=%r (drop_params=True) because %s does not support %s for it.", + ( + "" + if has_catalog_entry(model, custom_llm_provider) + else " If the deployment is a reasoning model, set model_info.base_model to its underlying model." + ), + ) + ) + + +def _warns_on_dropped_reasoning(value: object, model: str, custom_llm_provider: str | None) -> bool: + return not (_turns_reasoning_off(value) and is_known_non_reasoning_model(model, custom_llm_provider)) + + def get_optional_params( # use the openai defaults # https://platform.openai.com/docs/api-reference/chat/create @@ -5233,19 +5285,30 @@ def get_optional_params( verbose_logger.debug( "\nLiteLLM: Non-Default params passed to completion() %s", _debug_params_repr(non_default_params) ) - unsupported_params: Final = {} - for k in non_default_params: - if k not in supported_params: - if k in PROVIDER_UNVALIDATED_PARAMS: - continue - if k == "n" and n == 1: # langchain sends n=1 as a default value - continue # skip this param - unsupported_params[k] = non_default_params[k] + unsupported_params: Final[MappingProxyType[str, object]] = MappingProxyType( + { + k: non_default_params[k] + for k in non_default_params + if k not in supported_params and k not in PROVIDER_UNVALIDATED_PARAMS and not (k == "n" and n == 1) + } + ) if unsupported_params: if litellm.drop_params is True or (drop_params is not None and drop_params is True): for k in unsupported_params: non_default_params.pop(k, None) + for reasoning_param in _DROPPED_REASONING_PARAMS: + if reasoning_param in unsupported_params and _warns_on_dropped_reasoning( + unsupported_params[reasoning_param], base_model or model, custom_llm_provider + ): + verbose_logger.warning( + _dropped_reasoning_warning(base_model or model, custom_llm_provider), + reasoning_param, + unsupported_params[reasoning_param], + model, + custom_llm_provider, + reasoning_param, + ) else: raise UnsupportedParamsError( status_code=500, @@ -6761,6 +6824,8 @@ def get_model_info_helper( custom_llm_provider: str | None = None, api_base: str | None = None, api_key: str | None = None, + *, + catalog_only: bool = False, ) -> ModelInfoBase: """ Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's @@ -6794,7 +6859,7 @@ def get_model_info_helper( model_cost_custom_llm_provider: Final = custom_llm_provider ######################### provider_config: BaseLLMModelInfo | None = None - if custom_llm_provider and custom_llm_provider in LlmProvidersSet: + if not catalog_only and custom_llm_provider and custom_llm_provider in LlmProvidersSet: provider_config = ProviderConfigManager.get_provider_model_info( model=model, provider=LlmProviders(custom_llm_provider) ) @@ -6818,7 +6883,7 @@ def get_model_info_helper( e, ) - if custom_llm_provider == "huggingface": + if not catalog_only and custom_llm_provider == "huggingface": max_tokens: Final = _get_max_position_embeddings(model_name=model) return ModelInfoBase( key=model, diff --git a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py index a82dd98e31d..2d9381ec16a 100644 --- a/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_gpt_reasoning_transformation.py @@ -1,3 +1,4 @@ +import logging from collections.abc import Mapping from typing import Final @@ -279,6 +280,29 @@ def test_azure_gpt5_reasoning_effort_none_dropped(config: AzureOpenAIGPT5Config) assert "reasoning_effort" not in params or params.get("reasoning_effort") != "none" +@pytest.mark.parametrize(("model", "names_base_model"), [("azure/litellm-qa-sol-alias", True), ("azure/gpt-5", False)]) +def test_azure_dropped_effort_warning_suggests_base_model_only_for_a_name_the_map_lacks( + config: AzureOpenAIGPT5Config, caplog: pytest.LogCaptureFixture, model: str, names_base_model: bool +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "none"}, + optional_params={}, + model=model, + drop_params=True, + api_version="2025-04-01-preview", + declared_reasoning_efforts=(), + ) + + assert "reasoning_effort" not in params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort='none'" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.reasoning_effort_levels" in drop_warnings[0] + assert ("model_info.base_model" in drop_warnings[0]) is names_base_model + + # Logprobs support tests for Azure GPT-5.2 def test_azure_gpt5_2_supports_logprobs(config: AzureOpenAIGPT5Config): """Test that Azure GPT-5.2 models support logprobs parameters. diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 154797bab5a..2d75ce1f22f 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -1,6 +1,7 @@ import base64 import copy import json +import logging import os import re from collections.abc import Iterator, Mapping @@ -966,7 +967,9 @@ def test_reasoning_effort_maps_to_reasoning_effort_for_openai_gpt5_converse(mode "us.openai.gpt-6-astra", ], ) -def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, local_model_cost_map): +def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map( + model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: config = AmazonConverseConfig() assert litellm.utils.is_explicitly_disabled_factory( model=model, custom_llm_provider="bedrock_converse", key="supports_minimal_reasoning_effort" @@ -980,15 +983,19 @@ def test_openai_gpt5_converse_rejects_effort_level_disabled_in_model_map(model, drop_params=False, ) - optional_params = config.map_openai_params( - non_default_params={"reasoning_effort": "minimal"}, - optional_params={}, - model=model, - drop_params=True, - ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = config.map_openai_params( + non_default_params={"reasoning_effort": "minimal"}, + optional_params={}, + model=model, + drop_params=True, + ) _, additional_request_params, _, _ = config._prepare_request_params(optional_params, model) assert "reasoning" not in additional_request_params assert "thinking" not in additional_request_params + dropped: Final = [r.getMessage() for r in caplog.records if "Dropping reasoning_effort='minimal'" in r.getMessage()] + assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] @pytest.mark.parametrize( diff --git a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py index 6da131f38cc..a1466cfdae4 100644 --- a/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py +++ b/tests/unit/llms/bedrock/responses/test_bedrock_openai_responses.py @@ -7,13 +7,17 @@ falls back to the Chat Completions bridge and rides Converse. import json import logging from importlib.resources import files +from typing import Final from unittest.mock import patch import pytest import litellm from litellm.llms.bedrock.common_utils import bedrock_supports_openai_responses -from litellm.llms.bedrock.responses.transformation import BedrockOpenAIResponsesConfig +from litellm.llms.bedrock.responses.transformation import ( + BedrockOpenAIResponsesConfig, + _without_disabled_reasoning_effort, +) from litellm.responses.file_search.emulated_handler import should_use_emulated_file_search from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders @@ -337,13 +341,37 @@ class TestDisabledReasoningEffort: ) @pytest.mark.parametrize("model", ["us.openai.gpt-5.6-luna", MODEL]) - def test_effort_level_disabled_in_model_map_is_dropped_with_drop_params(self, model, local_model_cost_map): - params = _cfg().map_openai_params( - response_api_optional_params={"reasoning": {"effort": "minimal", "summary": "auto"}, "max_output_tokens": 64}, - model=model, - drop_params=True, - ) + def test_effort_level_disabled_in_model_map_is_dropped_with_a_warning( + self, model: str, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = _cfg().map_openai_params( + response_api_optional_params={ + "reasoning": {"effort": "minimal", "summary": "auto"}, + "max_output_tokens": 64, + }, + model=model, + drop_params=True, + ) assert params == {"reasoning": {"summary": "auto"}, "max_output_tokens": 64} + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] + assert len(dropped) == 1 and model in dropped[0] + assert "model_info" not in dropped[0] + + def test_bedrock_disabled_effort_drop_warns_on_its_own( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = _without_disabled_reasoning_effort( + {"reasoning": {"effort": "minimal"}, "max_output_tokens": 64}, MODEL, True + ) + assert params == {"max_output_tokens": 64} + dropped: Final = [ + r.getMessage() for r in caplog.records if "Dropping reasoning.effort='minimal'" in r.getMessage() + ] + assert len(dropped) == 1 and MODEL in dropped[0] def test_effort_only_reasoning_is_removed_when_dropped(self, local_model_cost_map): params = _cfg().map_openai_params( diff --git a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py index a2289167ad4..0ae6a73968d 100644 --- a/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py +++ b/tests/unit/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py @@ -9,6 +9,7 @@ gate, the URL construction for both paths, and the shared Bearer auth. import copy import logging +from typing import Final import pytest from botocore.exceptions import ( @@ -572,6 +573,32 @@ class TestBedrockMantleServiceTier: assert "priority" in str(mock_warning.call_args) +class TestBedrockMantleRefusedEffort: + def test_dropping_a_refused_effort_does_not_suggest_declaring_levels_mantle_ignores( + self, local_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = BedrockMantleResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": refused}}, + model="openai.gpt-6.1-sol", + drop_params=True, + ) + + assert "reasoning" not in params + drop_warnings: Final = [ + record.getMessage() + for record in caplog.records + if f"Dropping reasoning.effort={refused!r}" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info" not in drop_warnings[0] + + class TestBedrockMantleReasoningSummary: @pytest.mark.parametrize("summary", ["concise", "detailed"]) def test_unsupported_reasoning_summary_dropped_when_drop_params_true(self, summary): diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index a185cdaf718..f5a33f87fd0 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,4 +1,5 @@ import json +import logging from collections.abc import Callable, Mapping from types import SimpleNamespace from typing import Final, cast @@ -19,6 +20,7 @@ from litellm.llms.hosted_vllm.responses.transformation import HostedVLLMResponse from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.responses.transformation import XAIResponsesAPIConfig from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( ImageGenerationPartialImageEvent, IncompleteDetails, @@ -2479,6 +2481,105 @@ class TestResponsesSurfaceSharesTheEffortRule: ) assert mapped["reasoning"] == {"summary": "auto"} + @pytest.mark.parametrize( + ("declared_reasoning_efforts", "suggests_declared_levels"), + [((), True), (None, False)], + ids=["responses route reads model_info levels", "compact route reads none"], + ) + def test_dropping_a_refused_effort_logs_one_warning_naming_only_the_fix_its_route_honors( + self, + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + declared_reasoning_efforts: tuple[str, ...] | None, + suggests_declared_levels: bool, + ) -> None: + refused: Final = next( + level + for level in ("none", "minimal", "low") + if litellm.model_cost["gpt-6.1-sol"].get(f"supports_{level}_reasoning_effort") is False + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = ResponsesAPIRequestUtils.get_optional_params_responses_api( + model="gpt-6.1-sol", + responses_api_provider_config=OpenAIResponsesAPIConfig(), + response_api_optional_params={"reasoning": {"effort": refused}}, + drop_params=True, + declared_reasoning_efforts=declared_reasoning_efforts, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert repr(refused) in drop_warnings[0] and "gpt-6.1-sol" in drop_warnings[0] + assert ("model_info.reasoning_effort_levels" in drop_warnings[0]) is suggests_declared_levels + assert "model_info.base_model" not in drop_warnings[0] + + def test_dropping_reasoning_for_a_model_without_supports_reasoning_logs_one_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'medium'", "gpt-4o", "supports_reasoning")) + + def test_turning_reasoning_off_for_a_model_without_supports_reasoning_drops_it_silently( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none", "summary": "auto"}}, + model="gpt-4o", + drop_params=True, + ) + + assert "reasoning" not in mapped + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + + def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + litellm.register_model( + { + "pricing-only-test-model": { + "litellm_provider": "openai", + "mode": "responses", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "none"}}, + model="pricing-only-test-model", + drop_params=True, + ) + + assert "reasoning" not in mapped + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "pricing-only-test-model")) + + def test_a_forwarded_effort_logs_no_drop_warning( + self, local_model_cost_map: None, caplog: pytest.LogCaptureFixture + ) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + mapped: Final = OpenAIResponsesAPIConfig().map_openai_params( + response_api_optional_params={"reasoning": {"effort": "medium"}}, + model="gpt-6.1-sol", + drop_params=True, + ) + + assert mapped["reasoning"] == {"effort": "medium"} + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] + @pytest.mark.parametrize("level", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) def test_gpt5_3_codex_forwards_exactly_the_effort_levels_its_map_row_allows(self, local_model_cost_map, level): """The gate turns a stale false into a local refusal, so the Responses-only codex row has to diff --git a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py index 4cdc674c774..807a43c5d9d 100644 --- a/tests/unit/llms/openai/test_gpt_reasoning_transformation.py +++ b/tests/unit/llms/openai/test_gpt_reasoning_transformation.py @@ -1,3 +1,4 @@ +import logging import re from collections.abc import Mapping from typing import Final @@ -1589,3 +1590,55 @@ def test_gpt_6_astra_takes_the_reasoning_series_request_shape(): assert "max_tokens" not in params assert params["reasoning_effort"] == "max" assert params["verbosity"] == "low" + + +@pytest.mark.parametrize( + ("custom_llm_provider", "level", "advice"), + [ + ("azure", "none", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("azure", "xhigh", frozenset(("model_info.reasoning_effort_levels", "model_info.base_model"))), + ("openai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("azure_ai", "xhigh", frozenset(("model_info.reasoning_effort_levels",))), + ("github_copilot", "xhigh", frozenset()), + ], +) +def test_a_reasoning_effort_the_level_gate_drops_logs_a_warning_naming_only_the_fixes_its_route_honors( + caplog: pytest.LogCaptureFixture, custom_llm_provider: str, level: str, advice: frozenset[str] +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider=custom_llm_provider, + reasoning_effort=level, + drop_params=True, + ) + + assert "reasoning_effort" not in params + drop_warnings: Final = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert repr(level) in drop_warnings[0] and "gpt-5.6-terra-ptu" in drop_warnings[0] + assert ( + frozenset( + fragment + for fragment in ("model_info.reasoning_effort_levels", "model_info.base_model") + if fragment in drop_warnings[0] + ) + == advice + ) + + +def test_a_reasoning_effort_the_level_gate_forwards_logs_no_drop_warning(caplog: pytest.LogCaptureFixture) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + params: Final = litellm.get_optional_params( + model="gpt-5.6-terra-ptu", + custom_llm_provider="azure", + reasoning_effort="medium", + drop_params=True, + ) + + assert params["reasoning_effort"] == "medium" + assert not [record for record in caplog.records if "reasoning_effort" in record.getMessage()] diff --git a/tests/unit/router_utils/test_reasoning_effort_capability.py b/tests/unit/router_utils/test_reasoning_effort_capability.py index d0564812c48..1d8ee338cbb 100644 --- a/tests/unit/router_utils/test_reasoning_effort_capability.py +++ b/tests/unit/router_utils/test_reasoning_effort_capability.py @@ -626,8 +626,9 @@ class TestResolveDeploymentReasoningEffort: _sent("turbo\nforged line", drop_params=True) assert [record.getMessage() for record in caplog.records] == [ - "reasoning_effort unrecognized sent as None (rounding up) to deployment deployment-1, " - "model group kimi-k3, model hosted_vllm/my-kimi" + "Dropping reasoning level unrecognized (drop_params=True) for deployment deployment-1, model group " + "kimi-k3, model hosted_vllm/my-kimi because its model_info.reasoning_effort_levels does not list that " + "level. List every level the deployment supports." ] def test_line_breaks_in_logged_names_cannot_forge_a_log_line(self, caplog: pytest.LogCaptureFixture) -> None: @@ -669,6 +670,27 @@ class TestResolveDeploymentReasoningEffort: "reasoning_effort_rounding": "up", } + def test_only_a_dropped_level_logs_a_warning(self, caplog: pytest.LogCaptureFixture) -> None: + verbose_logger.addHandler(caplog.handler) + verbose_logger.propagate = False + verbose_logger.setLevel(logging.INFO) + declared: Final = {"reasoning_effort_levels": ["low", "high"]} + + _sent("none", declared, drop_params=True) + _sent("medium") + _sent("default", declared, drop_params=True) + _sent("default", declared) + + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + logging.INFO, + logging.INFO, + ] + dropped: Final = caplog.records[0].getMessage() + assert "Dropping reasoning level none" in dropped + assert "model_info.reasoning_effort_levels" in dropped + class TestAzureGpt6SolAndLunaAdvertiseTheOpenAiLevels: @pytest.mark.parametrize( diff --git a/tests/unit/test_utils_get_optional_params.py b/tests/unit/test_utils_get_optional_params.py index 13a64577efb..05b776cb873 100644 --- a/tests/unit/test_utils_get_optional_params.py +++ b/tests/unit/test_utils_get_optional_params.py @@ -1,4 +1,5 @@ import json +import logging from typing import Final from unittest.mock import MagicMock, patch @@ -2097,3 +2098,146 @@ def test_supports_system_message_merges_system_prompt_into_user_message() -> Non ] assert map_system_message_pt(messages=messages) == [{"role": "user", "content": "Listen here! Hello there!"}] + + +def test_dropping_an_unsupported_reasoning_effort_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping reasoning_effort" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'none'", "terra-ptu", "model_info.base_model")) + + +@pytest.mark.parametrize("provider", ["ollama", "ollama_chat", "lemonade", "huggingface"]) +def test_dropping_thinking_for_an_unmapped_local_model_sends_no_request( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + outbound: Final = respx.MockRouter(assert_all_mocked=False) + monkeypatch.setattr( + litellm, "module_level_client", HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(outbound.handler))) + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="unmapped-local-model", custom_llm_provider=provider, thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + assert len([record for record in caplog.records if "Dropping thinking" in record.getMessage()]) == 1 + assert not outbound.calls + + +def test_dropping_an_unsupported_thinking_logs_one_warning( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="terra-ptu", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("'disabled'", "terra-ptu", "model_info.base_model")) + + +@pytest.mark.parametrize( + ("reasoning_kwargs", "warns"), + [ + ({"reasoning_effort": "none"}, False), + ({"thinking": {"type": "disabled"}}, False), + ({"reasoning_effort": "high"}, True), + ({"thinking": {"type": "enabled", "budget_tokens": 1024}}, True), + ], +) +def test_reasoning_drop_warns_unless_a_mapped_non_reasoning_model_was_asked_for_none( + local_model_cost_map: None, + caplog: pytest.LogCaptureFixture, + reasoning_kwargs: dict[str, object], + warns: bool, +) -> None: + litellm.register_model( + { + "azure/plain-chat-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "max_tokens": 1024, + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "supports_reasoning": False, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="plain-chat-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + assert any("Dropping" in record.getMessage() for record in caplog.records) is warns + + +@pytest.mark.parametrize("reasoning_kwargs", [{"reasoning_effort": "none"}, {"thinking": {"type": "disabled"}}]) +def test_turning_reasoning_off_warns_when_only_custom_pricing_describes_the_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture, reasoning_kwargs: dict[str, object] +) -> None: + litellm.register_model( + { + "azure/pricing-only-test-deployment": { + "litellm_provider": "azure", + "mode": "chat", + "input_cost_per_token": 0.000001, + "output_cost_per_token": 0.000002, + } + } + ) + + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="pricing-only-test-deployment", custom_llm_provider="azure", drop_params=True, **reasoning_kwargs + ) + + assert not set(reasoning_kwargs) & set(optional_params) + drop_warnings: Final = [record.getMessage() for record in caplog.records if "Dropping" in record.getMessage()] + assert len(drop_warnings) == 1 + assert all(fragment in drop_warnings[0] for fragment in ("pricing-only-test-deployment", "model_info.base_model")) + + +def test_dropping_reasoning_for_a_catalog_reasoning_model_does_not_suggest_base_model( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + assert litellm.model_cost["azure/gpt-5.6-sol"].get("supports_reasoning") is True + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-5.6-sol", custom_llm_provider="azure", thinking={"type": "disabled"}, drop_params=True + ) + + assert "thinking" not in optional_params + drop_warnings: Final = [ + record.getMessage() for record in caplog.records if "Dropping thinking" in record.getMessage() + ] + assert len(drop_warnings) == 1 + assert "model_info.base_model" not in drop_warnings[0] + + +def test_turning_reasoning_off_for_a_catalog_non_reasoning_model_drops_it_silently( + local_model_cost_map: None, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger="LiteLLM"): + optional_params: Final = get_optional_params( + model="gpt-4o", custom_llm_provider="azure", reasoning_effort="none", drop_params=True + ) + + assert "reasoning_effort" not in optional_params + assert not [record for record in caplog.records if "Dropping" in record.getMessage()] From 67ae3fb527350caad8bd4463a5a37c9fbd8a67f4 Mon Sep 17 00:00:00 2001 From: Priyansh Nandwana <136072927+msdianprince-7@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:17:18 +0530 Subject: [PATCH 3/3] fix(websearch)!: keep the provider prefix on the agentic follow-up model (#38885) * fix(websearch): keep the provider prefix on the agentic follow-up model The agentic follow-up rebuilt its model string with "a slash means it is already qualified". Handlers are handed the provider-stripped model, and for a provider that routes through a sub-path the remainder still holds a slash, so bedrock/mantle/anthropic.claude-sonnet-5 arrived as mantle/anthropic.claude-sonnet-5 and stayed that way. The follow-up acompletion could not resolve a provider from it and raised "LLM Provider NOT provided". Prefix unless the model is already qualified with that same provider, in one shared helper used by both the handler and the websearch patch builder, which carried copies of the same heuristic. This also fixes openrouter, whose stripped model keeps a slash for the same reason, and the old startswith test that read openai as a prefix of openai_like. Fixes #38829 * test(websearch): cover the agentic follow-up call site The helper had unit tests but the call site did not, so the two changed lines in _execute_chat_completion_agentic_plan were uncovered. Drive the real follow-up with mock_response instead of patching litellm, which keeps the test on the behaviour: against the old code the sub-path case raises "LLM Provider NOT provided", and the ordinary-model case passes either way as a guard. * refactor(websearch): trim the comments on the provider-qualify helper * refactor(websearch): cut the helper and test docstrings to one line * fix(websearch): leave a hook-qualified follow-up model on its own provider A hook that patches in its own model owns that choice, so re-qualifying it sent a cross-provider follow-up to the original request's provider instead. Only a bare patched model and the request's provider-stripped model take the prefix now, and the sibling agentic-loop path resolves the model the same way. * fix(websearch): fall back to the request model when a patch carries no model An empty patched model dispatched a provider-only name instead of falling back, and a provider-stripped sub-path copied into the patch lost its prefix. A patched model now keeps its own provider only when its leading segment names one. * fix(websearch): keep the provider prefix on the responses follow-up model too The responses patch builder still used the old "a slash means already qualified" test, so a provider-stripped sub-path model reached the follow-up unqualified on that surface. It now resolves the model the same way the chat completion builder does. * fix(websearch): read custom_llm_provider without a member call on untyped kwargs * fix(websearch): type the chat patch builder kwargs instead of working around them * test(websearch): run the real search path and flatten the qualify helper tests * fix(websearch): route a hook's model on a JSON-configured provider and keep the request provider on an echoed model * fix(websearch): leave a follow-up model alone when its prefix already routes to the request provider * fix(websearch): send the agentic follow-up to the model the caller requested completion() and responses() record the caller's model before routing strips it, and the chat and Responses follow-up builders prefer that model. This keeps bedrock/converse_like/, bedrock/claude_platform/ and openrouter/openrouter/* deployments on their own route for the follow-up. * test(websearch): cover the recorded caller model in Headroom's retrieval follow-up * test(websearch): cover the recorded caller model in Compresr's retrieval follow-up Also merges the repeated provider-prefix check in core_helpers into one helper and drops a test docstring * test(websearch): drive the real search path through a mocked Perplexity HTTP route * refactor(websearch): read the provider from kwargs inside _caller_model * test(http-handler): type the agentic follow-up provider tests * refactor(websearch): record agentic loop params through a Logging method Moving the recording out of completion() keeps it under basedpyright's code-flow complexity limit after the merge with main. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../websearch_interception/handler.py | 30 +++-- .../chat_completion_agentic_loop.py | 5 +- litellm/litellm_core_utils/core_helpers.py | 43 +++++++ litellm/litellm_core_utils/litellm_logging.py | 4 + litellm/llms/custom_httpx/llm_http_handler.py | 10 +- litellm/main.py | 18 +++ litellm/responses/main.py | 2 + litellm/types/utils.py | 4 +- .../websearch_interception/conftest.py | 18 +++ .../test_websearch_chat_completion.py | 60 +++++++++- .../test_websearch_responses.py | 111 ++++++++++++++++++ .../test_websearch_rich_query_shape.py | 1 + .../test_chat_completion_agentic_loop.py | 43 +++++++ .../litellm_core_utils/test_core_helpers.py | 83 +++++++++++++ .../custom_httpx/test_llm_http_handler.py | 57 +++++++++ .../guardrail_hooks/test_compresr.py | 57 ++++++++- .../guardrail_hooks/test_headroom.py | 68 +++++++++++ 17 files changed, 592 insertions(+), 22 deletions(-) create mode 100644 tests/unit/integrations/websearch_interception/conftest.py diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 6e1ba3eea08..c25fcfe3e70 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -114,6 +115,18 @@ class _WebSearchSettingsView(TypedDict): websearch_interception_params: WebSearchInterceptionConfig +def _caller_model(model: str, logging_obj: "LiteLLMLoggingObj | None", kwargs: Mapping[str, object]) -> str: + agentic_view: Final[_AgenticLoopParamsView] = { + "agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {}) if logging_obj else {} + } + recorded_model: Final = agentic_view["agentic_loop_params"].get("model") + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + return qualify_provider_stripped_model( + recorded_model if isinstance(recorded_model, str) and recorded_model else model, + custom_llm_provider if isinstance(custom_llm_provider, str) else "", + ) + + class _SearchToolLitellmParams(TypedDict, total=False): search_provider: ReadOnly[str | None] @@ -1155,6 +1168,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, response_format=response_format, ) return AgenticLoopPlan( @@ -1181,6 +1195,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=optional_params, kwargs=kwargs, + logging_obj=logging_obj, ) return AgenticLoopPlan( run_agentic_loop=True, @@ -1195,6 +1210,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls: list[dict], optional_params: dict, kwargs: dict, + logging_obj: "LiteLLMLoggingObj | None", ) -> AgenticLoopRequestPatch: """Execute litellm.asearch() and build a Responses API rerun patch.""" search_tasks: Final = [ @@ -1254,9 +1270,7 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1814,6 +1828,7 @@ class WebSearchInterceptionLogger(CustomLogger): tool_calls=tool_calls, optional_params=dict[str, object](optional_params), kwargs=dict[str, object](kwargs), + logging_obj=logging_obj, response_format=response_format, ) if request_patch.messages is None: @@ -1836,7 +1851,8 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], + logging_obj: "LiteLLMLoggingObj | None", response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1922,11 +1938,7 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + full_model_name: Final = _caller_model(model, logging_obj, kwargs) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 55025508e40..8a53f0cbb42 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -34,6 +34,49 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + return _leads_with_provider(model) and inferred_provider(model) == custom_llm_provider + + +def _leads_with_provider(model: str) -> bool: + leading_segment, separator, _ = model.partition("/") + return bool(separator) and names_a_provider(leading_segment) + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + if _leads_with_provider(patch_model): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f4b069ded6..77c4136efda 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -165,6 +165,7 @@ from litellm.types.rerank import RerankResponse from litellm.types.router import RouterRateLimitError from litellm.types.utils import ( DEPLOYMENT_SCOPED_PRICING_FIELDS, + AgenticLoopParams, CachingDetails, CallTypes, CostBreakdown, @@ -1158,6 +1159,9 @@ class Logging(LiteLLMLoggingBaseClass): if "custom_llm_provider" in self.model_call_details: self.custom_llm_provider = self.model_call_details["custom_llm_provider"] + def record_agentic_loop_params(self, params: AgenticLoopParams) -> None: + self.model_call_details["agentic_loop_params"] = params + def record_applied_timeout(self, timeout: float | Timeout) -> None: self.applied_timeout = timeout self.litellm_params["timeout"] = timeout diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 9700465ebc3..9e125db4211 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -50,7 +50,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -6275,9 +6279,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) diff --git a/litellm/main.py b/litellm/main.py index a3a63d5a83a..d1af2de3a13 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -155,6 +155,7 @@ from litellm.types.litellm_params import ControlOptions, RetryStrategy from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( + AgenticLoopParams, BaseUrlRoutingHint, CustomPricingLiteLLMParams, ModelResponseStream, @@ -1272,6 +1273,16 @@ def _get_router_deployment_id(kwargs: dict) -> str | None: return None +def _record_agentic_loop_params( + logging_obj: object, requested_model: str, custom_llm_provider: str, owns_agentic_loop: bool +) -> None: + if not owns_agentic_loop or not isinstance(logging_obj, LiteLLMLoggingObj): + return + logging_obj.record_agentic_loop_params( + AgenticLoopParams(model=requested_model, custom_llm_provider=custom_llm_provider) + ) + + def _register_custom_pricing_for_request( model: str, custom_llm_provider: str, @@ -5691,6 +5702,7 @@ def completion( **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, "extra_headers": extra_headers, } + requested_model: Final = model model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5702,6 +5714,12 @@ def completion( else None ), ) + _record_agentic_loop_params( + logging_obj=logging, + requested_model=requested_model, + custom_llm_provider=custom_llm_provider, + owns_agentic_loop=not skip_responses_api_bridge, + ) deployment_model_info: Final = str_keyed_mapping( cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 5bed6037fa4..11e1602e9f8 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -90,6 +90,7 @@ from litellm.types.responses.streaming_websocket import ResponsesWebSocketReques from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, + AgenticLoopParams, CallTypes, CostPerToken, CustomPricingLiteLLMParams, @@ -1580,6 +1581,7 @@ def responses( local_vars=local_vars, use_chat_completions_api=use_chat_completions_api, ) + litellm_logging_obj.record_agentic_loop_params(AgenticLoopParams(model=model)) from litellm.litellm_core_utils.core_helpers import attribute_or_none diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 266a6a20131..9c3329d6387 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -263,10 +263,10 @@ class AgenticLoopParams(TypedDict, total=False): """ model: str - """The model string with provider prefix (e.g., 'bedrock/invoke/...')""" + """The model as the caller requested it, usually with its provider prefix (e.g., 'bedrock/invoke/...')""" custom_llm_provider: str - """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + """The LLM provider name (e.g., 'bedrock', 'anthropic'), when the recorder resolved one""" class OffPeakWindow(TypedDict, total=False): diff --git a/tests/unit/integrations/websearch_interception/conftest.py b/tests/unit/integrations/websearch_interception/conftest.py new file mode 100644 index 00000000000..a9f79a60bfe --- /dev/null +++ b/tests/unit/integrations/websearch_interception/conftest.py @@ -0,0 +1,18 @@ +from typing import Final + +import pytest +import respx + +import litellm + +PERPLEXITY_SEARCH_URL: Final = "https://api.perplexity.ai/search" + + +@pytest.fixture +def perplexity_search(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> respx.Route: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "test-perplexity-key") + monkeypatch.delenv("PERPLEXITY_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return respx_mock.post(PERPLEXITY_SEARCH_URL).respond( + json={"results": [{"title": "AI news", "url": "https://example.com/ai-news", "snippet": "AI news."}]} + ) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py index 21e50561f57..fc415374096 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_chat_completion.py @@ -5,15 +5,19 @@ Tests the end-to-end flow of websearch_interception callback with litellm.acompletion() for transparent server-side web search execution. """ -from unittest.mock import MagicMock +import datetime +from typing import Final +from unittest.mock import AsyncMock, MagicMock, patch import pytest +import respx import litellm from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) -from litellm.types.utils import LlmProviders, ModelResponse +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.types.utils import CallTypes, LlmProviders, ModelResponse @pytest.fixture @@ -378,7 +382,6 @@ async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): optional_params into the follow-up params without explicit removal causes the model to call the search tool again instead of synthesizing an answer. """ - from unittest.mock import patch websearch_logger = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.OPENAI]) @@ -422,3 +425,54 @@ async def test_execute_chat_completion_agentic_loop_strips_tool_choice(): if __name__ == "__main__": # Run with: pytest test_websearch_chat_completion.py -v -s pytest.main([__file__, "-v", "-s"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "requested_model,handler_model,custom_llm_provider", + [ + ("openrouter/openrouter/free", "openrouter/free", "openrouter"), + ("bedrock/converse_like/my-endpoint-model", "my-endpoint-model", "bedrock"), + ], +) +async def test_chat_followup_reuses_the_model_the_caller_requested( + requested_model: str, handler_model: str, custom_llm_provider: str, perplexity_search: respx.Route +) -> None: + """The provider handler sees the model after its routing prefixes are stripped, which + re-qualifying with the provider alone cannot restore.""" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.completion.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-chat-followup", + function_id="fn-chat-followup", + ) + litellm.completion( + model=requested_model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + logger: Final = WebSearchInterceptionLogger() + + plan: Final = await logger.async_build_chat_completion_agentic_loop_plan( + tools={ + "tool_calls": [{"id": "call_abc", "name": "litellm_web_search", "input": {"query": "latest ai news"}}], + "tool_type": "websearch", + "provider": custom_llm_provider, + "response_format": "openai", + }, + model=handler_model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=MagicMock(), + optional_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={"custom_llm_provider": custom_llm_provider}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + assert perplexity_search.call_count == 1 diff --git a/tests/unit/integrations/websearch_interception/test_websearch_responses.py b/tests/unit/integrations/websearch_interception/test_websearch_responses.py index 00b7299cc31..6ca812a5e40 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_responses.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_responses.py @@ -6,17 +6,22 @@ tool calls returned by /v1/responses, executes the search server-side, and builds a Responses-format follow-up request. """ +import datetime from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +import litellm + from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.types.integrations.custom_logger import ( RESPONSES_AGENTIC_SURFACE, ) +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.types.utils import CallTypes, LlmProviders @@ -252,6 +257,112 @@ async def test_build_responses_plan_produces_responses_input(): assert patch_obj.model == "openai/gpt-4o" +_WEB_SEARCH_FUNCTION_CALL: Final = { + "tool_calls": [ + { + "id": "fc_1", + "call_id": "fc_1", + "type": "function_call", + "name": "litellm_web_search", + "arguments": '{"query": "latest ai news"}', + "input": {"query": "latest ai news"}, + } + ], + "tool_type": "websearch", + "response_format": "responses", +} + + +async def _responses_followup_model(model: str, custom_llm_provider: str, logging_obj: Logging | None) -> str | None: + logger: Final = WebSearchInterceptionLogger() + plan: Final = await logger.async_build_responses_agentic_loop_plan( + tools={**_WEB_SEARCH_FUNCTION_CALL, "provider": custom_llm_provider}, + model=model, + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=_responses_output_with_web_search(), + optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}]}, + logging_obj=logging_obj, + stream=False, + kwargs={ + "custom_llm_provider": custom_llm_provider, + "_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE, + }, + ) + assert plan.request_patch is not None + return plan.request_patch.model + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_build_responses_plan_keeps_the_provider_on_a_sub_path_model() -> None: + """#38829 on the Responses surface: a provider-stripped model whose remainder still holds a + slash was left unqualified, so the follow-up could not resolve a provider.""" + followup_model: Final = await _responses_followup_model("mantle/anthropic.claude-sonnet-5", "bedrock", None) + + assert followup_model == "bedrock/mantle/anthropic.claude-sonnet-5" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_responses_followup_reuses_the_model_the_caller_requested() -> None: + """An OpenRouter model whose own id starts with "openrouter/" reaches the handler as + "openrouter/free", which re-qualifying alone cannot tell apart from a request for model "free".""" + requested_model: Final = "openrouter/openrouter/free" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-responses-followup", + function_id="fn-responses-followup", + ) + litellm.responses( + model=requested_model, + input="What's the latest AI news?", + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + + followup_model: Final = await _responses_followup_model("openrouter/free", "openrouter", logging_obj) + + assert followup_model == requested_model + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("perplexity_search") +async def test_responses_followup_keeps_the_caller_model_across_the_chat_completions_bridge() -> None: + """A Responses request bridged to chat completions re-enters completion() with the stripped model.""" + requested_model: Final = "openrouter/openrouter/free" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-bridged-followup", + function_id="fn-bridged-followup", + ) + litellm.responses( + model=requested_model, + input="What's the latest AI news?", + mock_response="searching", + litellm_logging_obj=logging_obj, + ) + litellm.completion( + model="openrouter/free", + custom_llm_provider="openrouter", + messages=[{"role": "user", "content": "What's the latest AI news?"}], + mock_response="searching", + litellm_logging_obj=logging_obj, + _skip_responses_api_bridge=True, + ) + + followup_model: Final = await _responses_followup_model("openrouter/free", "openrouter", logging_obj) + + assert followup_model == requested_model + + @pytest.mark.asyncio async def test_deployment_hook_converts_native_responses_web_search_tool(): """async_pre_call_deployment_hook converts a native Responses web_search tool diff --git a/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py b/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py index 72149e8a435..1020848ec89 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_rich_query_shape.py @@ -239,6 +239,7 @@ class TestCallSiteWiring: tool_calls=tool_calls, optional_params={}, kwargs={}, + logging_obj=None, ) call_kwargs = mock_asearch.await_args.kwargs diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 3cbc3591009..6167e91c37f 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -364,6 +364,49 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert logger.cleanup_calls == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model,custom_llm_provider,routed_model", + [ + ("cohere/command-a-03-2025", "cohere_chat", "command-a-03-2025"), + ("anthropic/claude-2", "anthropic_text", "claude-2"), + ], +) +async def test_sdk_followup_keeps_a_model_whose_prefix_already_routes_to_its_provider( + model: str, custom_llm_provider: str, routed_model: str +) -> None: + plan: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages()), + ) + gate: Final = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]}) + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-prefix", + function_id="fn-prefix", + dynamic_success_callbacks=[gate], + ) + + followup: Final = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={"mock_response": "done"}, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + stream=False, + ) + + assert isinstance(followup, ModelResponse) + assert followup.model == routed_model + assert followup._hidden_params["custom_llm_provider"] == custom_llm_provider + + @pytest.mark.asyncio async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs( restore_callbacks, diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 66a8e6bbda6..a4a3ae285b1 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -26,6 +26,8 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, preserve_upstream_non_openai_attributes, process_response_headers, + qualify_agentic_followup_model, + qualify_provider_stripped_model, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -637,6 +639,87 @@ class TestProviderResponseHeadersInHiddenParams: assert "additional_headers" not in sibling._hidden_params +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("mantle/anthropic.claude-sonnet-5", "bedrock", "bedrock/mantle/anthropic.claude-sonnet-5"), + ("invoke/anthropic.claude-v2", "bedrock", "bedrock/invoke/anthropic.claude-v2"), + ("openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), + ("gpt-4o", "openai", "openai/gpt-4o"), + ("claude-sonnet-4-5", "anthropic", "anthropic/claude-sonnet-4-5"), + ], +) +def test_the_provider_prefix_is_restored(model: str, provider: str, expected: str) -> None: + assert qualify_provider_stripped_model(model, provider) == expected + + +@pytest.mark.parametrize( + "model,provider", + [ + ("bedrock/mantle/anthropic.claude-sonnet-5", "bedrock"), + ("openai/gpt-4o", "openai"), + ("cohere/command-a-03-2025", "cohere_chat"), + ("ai21/jamba-1.5-large", "ai21_chat"), + ("anthropic/claude-2", "anthropic_text"), + ], +) +def test_an_already_qualified_model_is_left_alone(model: str, provider: str) -> None: + assert qualify_provider_stripped_model(model, provider) == model + + +def test_a_provider_that_only_shares_a_prefix_is_still_qualified() -> None: + assert qualify_provider_stripped_model("openai_like/foo", "openai") == "openai/openai_like/foo" + + +def test_no_provider_leaves_the_model_untouched() -> None: + assert qualify_provider_stripped_model("gpt-4o", "") == "gpt-4o" + + +def test_a_cross_provider_patched_model_is_dispatched_as_the_hook_asked() -> None: + assert qualify_agentic_followup_model("anthropic/claude-sonnet-4-5", "gpt-4o", "openai") == ( + "anthropic/claude-sonnet-4-5" + ) + + +def test_a_bare_patched_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("gpt-4o-mini", "gpt-4o", "openai") == "openai/gpt-4o-mini" + + +def test_a_sub_path_request_model_keeps_its_provider() -> None: + assert qualify_agentic_followup_model(None, "mantle/anthropic.claude-sonnet-5", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_an_unpatched_ordinary_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model(None, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_model_already_holding_the_request_provider_is_left_alone() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "gpt-4o", "openai") == "openai/gpt-4o" + + +@pytest.mark.parametrize("patched", ["", None]) +def test_a_patch_without_a_model_falls_back_to_the_request_model(patched: str | None) -> None: + assert qualify_agentic_followup_model(patched, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_sub_path_model_still_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("mantle/anthropic.claude-sonnet-5", "gpt-4o", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_a_patched_model_on_a_json_configured_provider_reaches_that_provider() -> None: + assert qualify_agentic_followup_model("aihubmix/gpt-4o", "gpt-4o", "openai") == "aihubmix/gpt-4o" + + +def test_a_patch_echoing_the_request_model_keeps_the_request_provider() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "openai/gpt-4o", "openrouter") == ( + "openrouter/openai/gpt-4o" + ) + + def test_spend_metadata_merge_survives_the_logging_thread_inserting_into_the_request_metadata() -> None: metadata: Final[dict[str, object]] = {} diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 90d1f6772a0..d8326536d03 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5634,6 +5634,63 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers) +class TestAgenticFollowUpKeepsTheProviderPrefix: + @staticmethod + async def _run_followup( + model: str, custom_llm_provider: str, patched_model: str | None = None + ) -> ModelResponse: + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, + ) + + response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "hi"}], model=patched_model + ), + ), + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"mock_response": "ok from followup"}, + kwargs={}, + custom_llm_provider=custom_llm_provider, + depth=0, + max_loops=2, + fingerprints=[], + fingerprint="fp", + ) + assert isinstance(response, ModelResponse) + return response + + @pytest.mark.asyncio + async def test_a_sub_path_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_an_ordinary_model_still_resolves_a_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o-mini", "openai") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self) -> None: + response: Final = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5") + + assert response.model == "claude-sonnet-4-5" + assert response._hidden_params["custom_llm_provider"] == "anthropic" + + @pytest.mark.asyncio + async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self) -> None: + response: Final = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") + + assert response._hidden_params["custom_llm_provider"] == "openrouter" + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py index feb7090c2e3..cc5ad10b645 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_compresr.py @@ -15,16 +15,20 @@ Tests cover: - fail_closed raises HTTPException; fail_open forwards uncompressed """ +import datetime import hashlib import json from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, create_autospec, patch import httpx import pytest from fastapi import HTTPException +import litellm from litellm.constants import PRE_CALL_EXECUTED_GUARDRAILS_KEY +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( COMPRESR_RETRIEVE_TOOL_NAME, CompresrGuardrail, @@ -33,7 +37,7 @@ from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( _scoped_store_key, has_compresr_retrieve_tool, ) -from litellm.types.utils import GenericGuardrailAPIInputs +from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs FAKE_API_BASE = "https://compresr.example.com" FAKE_API_KEY = "cmp_test-key" @@ -1435,6 +1439,57 @@ async def test_agentic_plan_builds_responses_followup_shape( assert output_item["output"] == TOOL_OUTPUT +@pytest.mark.asyncio +async def test_responses_retrieval_followup_reuses_the_model_the_caller_requested( + guardrail: CompresrGuardrail, +) -> None: + requested_model: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" + hash_value: Final = "f" * 24 + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-1", + function_id="fn-compresr-followup", + ) + litellm.responses( + model=requested_model, + input=USER_QUESTION, + mock_response="retrieving", + litellm_logging_obj=logging_obj, + ) + guardrail._store_originals(_scoped_store_key(logging_obj), {hash_value: TOOL_OUTPUT}) + response: Final = MagicMock() + response.output = [{"type": "function_call", "call_id": "fc_1"}] + response.content = None + + plan: Final = await guardrail.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "fc_1", + "type": "function", + "name": COMPRESR_RETRIEVE_TOOL_NAME, + "arguments": {"hash": hash_value}, + } + ] + }, + model="converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": USER_QUESTION}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + + @pytest.mark.asyncio async def test_agentic_plan_chat_parallel_tool_calls_echoes_only_retrieve( guardrail: CompresrGuardrail, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py index 3336b20f5eb..741fea5f2b9 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_headroom.py @@ -21,6 +21,7 @@ Tests cover: loop resolves the retrieve tool call, then fake-streamed back to the client """ +import datetime import json import time from typing import Final, Optional @@ -33,6 +34,7 @@ from fastapi import HTTPException import litellm +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( HeadroomGuardrail, has_headroom_retrieve_tool, @@ -771,6 +773,72 @@ async def test_async_build_agentic_loop_plan_builds_responses_api_function_call_ assert output_item["output"] == original_content +@pytest.mark.asyncio +async def test_responses_retrieval_followup_reuses_the_model_the_caller_requested( + guardrail: HeadroomGuardrail, +) -> None: + requested_model: Final = "bedrock/converse/us.anthropic.claude-sonnet-5-5" + retrieve_hash: Final = "b573993006976af767214fac" + logging_obj: Final = Logging( + model=requested_model, + messages=[], + stream=False, + call_type=CallTypes.responses.value, + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-1", + function_id="fn-headroom-followup", + ) + litellm.responses( + model=requested_model, + input=f"What does it say? hash={retrieve_hash}", + mock_response="retrieving", + litellm_logging_obj=logging_obj, + ) + response: Final = MagicMock() + response.choices = None + response.content = None + response.output = [ + { + "type": "function_call", + "id": "fc_abc123", + "call_id": "call_abc123", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": json.dumps({"hash": retrieve_hash}), + } + ] + guardrail._issued_hashes_by_call_id["call-1"] = (frozenset({retrieve_hash}), time.monotonic() + 999) + + with patch.object( + guardrail.async_handler, + "get", + new_callable=AsyncMock, + return_value=_make_retrieve_response("The full compressed content."), + ): + plan: Final = await guardrail.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_abc123", + "type": "function", + "name": HEADROOM_RETRIEVE_TOOL_NAME, + "arguments": {"hash": retrieve_hash}, + } + ] + }, + model="converse/us.anthropic.claude-sonnet-5-5", + messages=[{"role": "user", "content": f"What does it say? hash={retrieve_hash}"}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + kwargs={"litellm_call_id": "call-1"}, + ) + + assert plan.request_patch is not None + assert plan.request_patch.model == requested_model + + @pytest.mark.asyncio async def test_async_build_agentic_loop_plan_builds_anthropic_tool_result_messages( guardrail: HeadroomGuardrail,