diff --git a/litellm/constants.py b/litellm/constants.py index 7f2d98b7a28..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 6c0e463bd23..75b0329315f 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -207,7 +208,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -218,7 +219,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/main.py b/litellm/main.py index 996e93589fe..a3a63d5a83a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5703,15 +5703,16 @@ def completion( ), ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5808,7 +5809,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50b3560f05b..aa00ea73f8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/litellm/utils.py b/litellm/utils.py index aeb25f81841..adf195668ee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -3501,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50b3560f05b..aa00ea73f8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3070ebb0235..ed7dd5ec146 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -378,6 +378,20 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +@pytest.fixture +def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable[[str, bool], None]: + def declare(model_cost_key: str, supported: bool) -> None: + entry: Final = { + "litellm_provider": model_cost_key.split("/", 1)[0], + "mode": "chat", + "supports_mid_conversation_system": supported, + } + monkeypatch.setitem(litellm.model_cost, model_cost_key, entry) + litellm.get_model_info.cache_clear() + + return declare + + NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 53df6610913..f6dae2c304f 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -783,3 +783,26 @@ def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mo assert request.url.params["api-version"] == "2024-12-01-preview" assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} assert response.choices[0].message.content == "Hello" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("config", [AzureOpenAIConfig(), AzureOpenAIGPT5Config()]) +def test_azure_keeps_a_later_developer_message_in_place_as_system(config: AzureOpenAIConfig) -> None: + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="my-deployment", + custom_llm_provider="azure", + api_base="https://example-resource.openai.azure.com", + model_info=None, + ) + assert [message["role"] for message in messages] == ["system", "user", "assistant", "system", "user"] + assert messages[3]["content"] == "Answer with exactly one word." diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index ccac6ffc309..42dc222e420 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,5 +1,6 @@ import json import traceback +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -1098,3 +1099,25 @@ def test_map_azure_model_group(model_group_header, expected_model): config = AzureAICohereConfig() assert config._map_azure_model_group(model_group_header) == expected_model + + +def test_translate_developer_role_hoists_a_later_developer_message_for_a_model_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("azure_ai/system-first-model", False) + messages: Final = AzureAIStudioConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="system-first-model", + custom_llm_provider="azure_ai", + api_base="https://example.services.ai.azure.com/models", + model_info=None, + ) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user"] + assert messages[0]["content"] == "You are terse.\n\nAnswer with exactly one word." diff --git a/tests/unit/llms/base_llm/test_base_utils.py b/tests/unit/llms/base_llm/test_base_utils.py new file mode 100644 index 00000000000..7052fd04242 --- /dev/null +++ b/tests/unit/llms/base_llm/test_base_utils.py @@ -0,0 +1,494 @@ +from typing import Final + +from litellm.llms.base_llm.base_utils import ( + hoist_developer_messages_into_leading_system_message, + merge_system_message_runs, +) + + +class TestMergeSystemMessageRuns: + def test_later_developer_message_stays_in_place_while_the_leading_run_folds(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Sandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex\n\nSandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_run_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + + def test_hoisted_developer_message_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": 42}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": 42}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +class TestHoistDeveloperMessagesIntoLeadingSystemMessage: + def test_single_developer_message(self) -> None: + messages: Final = [ + {"role": "developer", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_developer_messages_merge(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Rule 1: Be concise."}, + {"role": "developer", "content": "Rule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rule 1: Be concise.\n\nRule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + + def test_system_followed_by_developer_merges(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base instructions."}, + {"role": "developer", "content": "Developer override."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base instructions.\n\nDeveloper override."}, + {"role": "user", "content": "Hello"}, + ] + + def test_developer_followed_by_system_merges(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Developer instruction."}, + {"role": "system", "content": "System instruction."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Developer instruction.\n\nSystem instruction."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_native_system_messages_merge(self) -> None: + messages: Final = [ + {"role": "system", "content": "System part 1."}, + {"role": "system", "content": "System part 2."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System part 1.\n\nSystem part 2."}, + {"role": "user", "content": "Hello"}, + ] + + def test_empty_string_side_does_not_pad(self) -> None: + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": ""}, {"role": "developer", "content": "Rules"}] + ) + ) == [{"role": "system", "content": "Rules"}] + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": "Rules"}, {"role": "developer", "content": ""}] + ) + ) == [{"role": "system", "content": "Rules"}] + + def test_empty_string_message_adds_no_block_when_the_run_merges_as_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": ""}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + + def test_two_cached_string_messages_keep_one_breakpoint_each(self) -> None: + messages: Final = [ + {"role": "system", "content": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"role": "system", "content": "B", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + }, + ] + + def test_cache_control_supplied_only_by_the_later_message_marks_only_its_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "A"}, + {"role": "developer", "content": "B", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A"}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + def test_cache_control_of_the_first_message_stays_on_its_own_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "developer", "content": "Additional instructions"}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAdditional instructions"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_cached_string_message_merged_with_block_content_keeps_its_breakpoint(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": [{"type": "text", "text": "Answer with exactly one word."}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_plain_messages_ahead_of_a_cached_one_keep_their_own_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "developer", "content": "Rule"}, + {"role": "developer", "content": "Cached", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Base"}, + {"type": "text", "text": "\n\nRule"}, + {"type": "text", "text": "\n\nCached", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_name_of_the_last_named_message_survives_a_long_run(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base", "name": "first"}, + *({"role": "developer", "content": f"Rule {index}"} for index in range(2000)), + {"role": "developer", "content": "Last", "name": "last"}, + {"role": "user", "content": "Hello"}, + ] + merged: Final = list(hoist_developer_messages_into_leading_system_message(messages)) + assert len(merged) == 2 + assert merged[0]["name"] == "last" + assert merged[0]["content"] == "\n\n".join(["Base", *(f"Rule {index}" for index in range(2000)), "Last"]) + + def test_billing_metadata_keeps_its_own_block_when_merged(self) -> None: + billing: Final = {"role": "system", "content": "x-anthropic-billing-header: cc_version=1.0;"} + advisory: Final = {"role": "system", "content": "Guardrail advisory: request was flagged"} + assert list(hoist_developer_messages_into_leading_system_message([billing, advisory])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + {"type": "text", "text": "\n\nGuardrail advisory: request was flagged"}, + ], + } + ] + assert list(hoist_developer_messages_into_leading_system_message([advisory, billing])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Guardrail advisory: request was flagged"}, + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + ], + } + ] + + def test_developer_message_after_a_user_turn_is_hoisted_into_the_leading_system_message(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "developer", "content": "Developer turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System turn 1\n\nDeveloper turn 2"}, + {"role": "user", "content": "User turn 1"}, + {"role": "user", "content": "User turn 2"}, + ] + + def test_hoisted_developer_message_keeps_only_the_fields_a_system_message_has(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word.", "name": "codex", "extra_key": "x"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word.", "name": "codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_developer_message_in_second_position_without_a_leading_system_message_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_every_later_developer_message_is_hoisted_in_order_across_assistant_and_tool_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "developer", "content": "Update B"}, + {"role": "user", "content": "Turn 3"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A\n\nUpdate B"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "user", "content": "Turn 3"}, + ] + + def test_developer_message_that_closes_the_conversation_after_an_assistant_turn_becomes_a_user_turn(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with the single word PONG and nothing else."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "user", "content": "Reply with the single word PONG and nothing else."}, + ] + + def test_closing_developer_message_keeps_its_cache_mark_as_a_user_turn(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with PONG.", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages))[-1] == { + "role": "user", + "content": "Reply with PONG.", + "cache_control": {"type": "ephemeral"}, + } + + def test_only_the_closing_developer_run_after_an_assistant_turn_becomes_user_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "developer", "content": "Closing B"}, + {"role": "developer", "content": "Closing C"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A"}, + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "user", "content": "Closing B"}, + {"role": "user", "content": "Closing C"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_user_turn_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Reply with exactly one word: the capital of France"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Reply with exactly one word: the capital of France"}, + {"role": "user", "content": "Hi there"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_tool_result_is_hoisted(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + {"role": "developer", "content": "Answer with exactly one word."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + ] + + def test_hoisted_single_text_block_developer_message_joins_string_instructions_as_text(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "developer", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex\n\n"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + + def test_multi_block_developer_message_keeps_its_blocks_after_string_instructions(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi"}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Codex"}, + {"type": "text", "text": "\n\nRule 1"}, + {"type": "text", "text": "Rule 2"}, + ], + }, + {"role": "user", "content": "Hi"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_system_message_without_content_folds_into_the_run_instead_of_raising(self) -> None: + messages: Final = [ + {"role": "system", "content": "Rules"}, + {"role": "system"}, + {"role": "system", "content": None}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1"}, + {"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "system", "content": "System turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == messages + + def test_list_content_merge_preserves_block_level_cache_control(self) -> None: + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}]}, + {"role": "developer", "content": [{"type": "text", "text": "Extra"}]}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nExtra"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 61e5545b8bf..28f4fdead9e 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -2358,3 +2359,86 @@ def test_global_disable_flag_with_transform_messages_helper( ) assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] + + +_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +def test_translate_developer_role_keeps_a_later_developer_message_in_place_for_an_undeclared_fireworks_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +def test_translate_developer_role_leaves_a_leading_system_and_developer_pair_unmerged_for_an_undeclared_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("model", ["qwen3p8-max", "accounts/fireworks/models/qwen3p8-max"]) +def test_translate_developer_role_hoists_a_later_developer_message_for_the_system_first_qwen3p8_model( + local_model_cost_map: None, model: str +) -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model=model, + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("deployment_declared", "expected_roles"), + [(False, ["system", "user", "user"]), (True, ["system", "user", "system", "user"])], +) +def test_translate_developer_role_follows_the_deployment_model_info_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], deployment_declared: bool, expected_roles: list[str] +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", not deployment_declared) + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/shared-backend", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info={"id": "deployment-a", "supports_mid_conversation_system": deployment_declared}, + ) + + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 878e0285cd1..7fc6ac2c436 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -1,5 +1,5 @@ import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final, TypedDict, cast from unittest.mock import MagicMock, patch @@ -7,6 +7,7 @@ from urllib.parse import quote import httpx import pytest +import respx from openai.types.responses import ( EasyInputMessage, ResponseFunctionToolCall, @@ -81,6 +82,14 @@ def _mock_http_client(response_body: Mapping[str, object]) -> MagicMock: return client +def _fireworks_responses_route(model: str) -> respx.Route: + return respx.post(FIREWORKS_RESPONSES_URL).mock(return_value=httpx.Response(200, json=_fireworks_response(model))) + + +def _routed_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + def _sent_request(client: MagicMock) -> tuple[str, Mapping[str, str], Mapping[str, object]]: kwargs: Final = client.post.call_args.kwargs body: Final = kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) @@ -182,6 +191,60 @@ def test_responses_call_folds_developer_items_into_instructions() -> None: ) +@respx.mock +def test_responses_call_keeps_developer_items_in_place_for_a_deployment_that_accepts_later_system_items() -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/accepts-later-system") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/accepts-later-system", + instructions="You are Codex", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model_info={"id": "deployment-1", "supports_mid_conversation_system": True}, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "You are Codex" + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, + {"role": "user", "content": "What is the capital of France?"}, + ) + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "closing_role"), + [(None, "system"), ({"id": "deployment-1", "supports_mid_conversation_system": False}, "user")], +) +def test_responses_call_sends_the_closing_developer_item_as_a_user_turn_only_for_a_declared_system_first_deployment( + model_info: Mapping[str, object] | None, closing_role: str +) -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/kimi-k3") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Reply with PONG."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "Answer with exactly one word." + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": closing_role, "content": "Reply with PONG.", "type": "message"}, + ) + + def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): @@ -667,3 +730,40 @@ def test_responses_call_forwards_litellm_user_id_as_user( _, _, body = _sent_request(client) assert body.get("user") == expected_user assert "fireworks_forward_user_id" not in body + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "expected_instructions", "expected_input"), + [ + (None, "Answer in one word.", ({"role": "user", "content": "What is the capital of France?"},)), + ( + {"id": "deployment-1", "supports_mid_conversation_system": True}, + None, + ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Answer in one word.", "type": "message"}, + ), + ), + ], +) +def test_responses_call_forwards_developer_items_in_place_only_when_the_deployment_itself_declares_true( + declare_mid_conversation_system: Callable[[str, bool], None], + model_info: Mapping[str, object] | None, + expected_instructions: str | None, + expected_input: tuple[Mapping[str, object], ...], +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", True) + route: Final = _fireworks_responses_route("accounts/fireworks/models/shared-backend") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/shared-backend", + input=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "developer", "content": "Answer in one word."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body.get("instructions") == expected_instructions + assert tuple(body["input"]) == expected_input diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..2f63ef070e6 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,9 +8,13 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm @@ -183,3 +187,92 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +_LATER_DEVELOPER_INPUT: Final = ( + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +_VLLM_RESPONSES_URL: Final = "https://test-vllm.example.com/v1/responses" + + +def _vllm_responses_route(output_text: str) -> respx.Route: + return respx.post(_VLLM_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_make_mock_responses_api_response(output_text)) + ) + + +def _sent_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + +@respx.mock +def test_hosted_vllm_responses_folds_a_later_developer_item_into_instructions_for_a_system_first_model( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("hosted_vllm/Qwen/Qwen3.6-27B", False) + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3.6-27B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == (_LATER_DEVELOPER_INPUT[0], _LATER_DEVELOPER_INPUT[2]) + + +@respx.mock +def test_hosted_vllm_responses_folds_for_a_deployment_declared_system_first_and_sends_a_closing_developer_as_user() -> None: + assistant_turn: Final = {"type": "message", "role": "assistant", "content": "Paris"} + route: Final = _vllm_responses_route("PONG") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=[*_LATER_DEVELOPER_INPUT, assistant_turn, {"role": "developer", "content": "Reply with PONG."}], + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == ( + _LATER_DEVELOPER_INPUT[0], + _LATER_DEVELOPER_INPUT[2], + assistant_turn, + {"role": "user", "content": "Reply with PONG.", "type": "message"}, + ) + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_later_developer_item_in_place_for_an_undeclared_model() -> None: + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant." + assert tuple(body["input"]) == _LATER_DEVELOPER_INPUT + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_developer_item_without_content_on_a_deployment_declared_system_first() -> None: + route: Final = _vllm_responses_route("Paris") + input_items: Final = ( + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"type": "message", "role": "developer"}, + ) + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input=list(input_items), + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + assert tuple(_sent_body(route)["input"]) == input_items diff --git a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 9b8788d3769..8a690a70a97 100644 --- a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -639,3 +639,25 @@ def test_litellm_gateway_from_sdk_with_thinking_param(respx_mock): ) e = exc_info.value assert "Connection error." in str(e) + + +def test_translate_developer_role_leaves_a_later_developer_message_in_place_for_the_downstream_proxy() -> None: + translated: Final = LiteLLMProxyChatConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="fireworks-qwen", + custom_llm_provider="litellm_proxy", + api_base="http://inner-proxy:4000", + model_info=None, + ) + + assert list(translated) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 70436662ef0..5b85a94b24d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,7 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ import base64 +import copy import json +from collections.abc import Callable from typing import Final import pytest @@ -1820,3 +1822,97 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: ) assert request["tools"][0] is tool + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("config", "custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + (OpenAIGPTConfig(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPT5Config(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", False, ["system", "user", "assistant", "user"]), + (OpenAIGPTConfig(), "groq", None, None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "groq", None, False, ["system", "user", "assistant", "user"]), + ], +) +def test_translate_developer_role_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + config: OpenAIGPTConfig, + custom_llm_provider: str, + api_base: str | None, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles + assert "Answer with exactly one word." in [message["content"] for message in messages if message["role"] == "system"][-1] + + +@pytest.mark.parametrize("role", [["user"], {"name": "user"}]) +@pytest.mark.parametrize( + ("declared", "expected_order"), + [(None, ("caller", "instruction")), (False, ("instruction", "caller"))], +) +def test_translate_developer_role_places_a_message_with_a_non_string_role_like_any_conversation_turn( + declare_mid_conversation_system: Callable[[str, bool], None], + role: object, + declared: bool | None, + expected_order: tuple[str, str], +) -> None: + if declared is not None: + declare_mid_conversation_system("groq/system-first-model", declared) + caller_turn: Final = {"role": role, "content": "Hi there"} + by_name: Final = {"caller": caller_turn, "instruction": {"role": "system", "content": "Answer in one word."}} + messages: Final = OpenAIGPTConfig().translate_developer_role_to_system_role( + messages=[caller_turn, {"role": "developer", "content": "Answer in one word."}], + model="system-first-model", + custom_llm_provider="groq", + api_base=None, + model_info=None, + ) + assert list(messages) == [by_name[name] for name in expected_order] + + +@pytest.mark.respx(assert_all_called=True) +def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_system_first_deployment( + respx_mock: respx.MockRouter, +) -> None: + chat: Final = respx_mock.post("http://vllm.internal:8000/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + messages: Final = [ + {"role": "user", "content": "Describe the image."}, + {"role": "assistant", "content": "Send it."}, + { + "role": "developer", + "content": [{"type": "text", "text": "Look at this."}, {"type": "image_url", "image_url": "data:image/png;base64,aW1hZ2U="}], + }, + ] + sent: Final = copy.deepcopy(messages) + litellm.completion( + model="openai/system-first-model", + api_base="http://vllm.internal:8000/v1", + api_key="sk-test", + messages=messages, + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + ) + assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" + assert messages == sent diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index 121fab66c15..e78e1a6f12b 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -181,3 +181,20 @@ def test_litellm_responses(): print("response: ", response) assert isinstance(response.usage.completion_tokens_details, CompletionTokensDetails) + + +def test_translate_developer_role_keeps_developer_messages_untouched() -> None: + messages: Final = [ + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert ( + OpenAIOSeriesConfig().translate_developer_role_to_system_role( + messages=messages, + model="my-reasoning-deployment", + custom_llm_provider="openai", + api_base=None, + model_info=None, + ) + == messages + ) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 11d7c7d6d3c..5b27a446fb3 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2,7 +2,7 @@ import asyncio import importlib import json import os -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncCursorPage, MessageData, OpenAIChatCompletion, + OpenAIConfig, OpenAIMessage as Message, Run, SyncCursorPage, @@ -2010,3 +2011,40 @@ async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_head assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} assert sent.headers["x-caller"] == "unit" assert sent.headers["authorization"] == "Bearer sk-test" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + ("openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", None, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", False, ["system", "user", "assistant", "user"]), + ], +) +def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + custom_llm_provider: str, + api_base: str, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = OpenAIConfig().translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..2870016a5e9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10869,6 +10869,44 @@ def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_e assert "cache_control_injection_points" not in optional_params +@pytest.mark.parametrize( + ("cost_map_declared", "deployment_declared", "expected"), + [ + (None, None, None), + (False, None, False), + (True, None, True), + (True, False, False), + (False, True, True), + (None, False, False), + ], +) +def test_declared_mid_conversation_system_support_prefers_the_deployment_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], + cost_map_declared: bool | None, + deployment_declared: bool | None, + expected: bool | None, +) -> None: + if cost_map_declared is not None: + declare_mid_conversation_system("hosted_vllm/mid-conversation-probe", cost_map_declared) + deployment_model_info: Final = ( + {"id": "deployment-1"} + if deployment_declared is None + else {"id": "deployment-1", "supports_mid_conversation_system": deployment_declared} + ) + assert ( + litellm.utils.declared_mid_conversation_system_support( + "mid-conversation-probe", "hosted_vllm", deployment_model_info + ) + is expected + ) + + +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") + litellm.get_model_info.cache_clear() + assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None + + def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy applies cache_control_injection_points itself, so the point must ride along in extra_body."""