From 71babd931bf2bf1823f73be25c78bc35194d8c15 Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Thu, 29 Oct 2026 04:12:13 -0700 Subject: [PATCH] fix(llms): move later developer messages for models declared system-first (#48897) * fix(transformation): merge consecutive system and developer messages for non-OpenAI providers and Responses API * fix(base_utils): keep later cache_control and billing metadata boundaries when merging system messages * perf(base_utils): fold consecutive system messages in a single linear pass The tuple accumulator passed to reduce copied the whole prefix on every step, making map_developer_role_to_system_role O(n^2) in the number of messages. Message count is not bounded by default, so a caller could tie up a proxy worker with thousands of small messages before any provider call. Use a list and merge into its last element in place instead. * fix(base_utils): hoist later developer messages into the leading system message for non-OpenAI providers A developer message that arrives after the first user turn (Codex CLI's per-turn permissions item, a re-injected instruction after compaction, a mid-conversation policy update) was translated to a system message in place, and system-first chat templates such as Fireworks' qwen3.8 reject any system message that is not at the beginning with 400. The fold that merges consecutive system messages never saw it because a user turn sat in between. Move every developer message that follows the leading system block to the end of that block before the fold, so it merges into the single leading system message; client-authored mid-conversation system messages stay where the client put them. * fix(openai): hoist later developer messages into one leading system message on OpenAI-compatible backends Developer role translation moves onto the provider config. OpenAI's own endpoint, Azure OpenAI, and the o-series keep their current behavior. Every other config inheriting OpenAIGPTConfig (Fireworks, DeepSeek, Together, vLLM, Groq, openai/ with a custom api_base) hoists developer messages that arrive after the leading system block into that block and folds the block into a single system message, which is what system-first chat templates require. Providers with their own message format are untouched. Messages carrying cache_control or Anthropic billing metadata fold as their own text blocks, so breakpoints and provider-side stripping survive the merge. The Responses bridge goes back to prepending instructions as a separate system message and lets the hoist fold it. * refactor(base_utils): fold each run of system messages in one pass The pairwise fold recopied the accumulated system text or block list on every merge, so a request carrying many consecutive system or developer messages cost quadratic time in preprocessing. Group consecutive system messages once and build the merged message from the whole run: plain strings join with a blank line, and any cached, billing, or block content keeps one text block per message. The name of the last named message in the run survives, as before. * fix(base_utils): keep a closing developer message in place and hoist for OpenAIConfig backends A developer message that closes the conversation stays where the client put it, translated to system in place, so the request never ends on the assistant's turn. OpenAIConfig and its subclasses (Azure AI Foundry, GitHub Copilot, ChatGPT, RAGFlow) now share the OpenAI-compatible hoist, litellm_proxy passes developer messages through to the downstream proxy, and the Anthropic billing header literals use the shared constant * fix(base_utils): keep a closing developer message in place only after an assistant turn A developer message that closes the conversation after a user or tool message hoists like any other, so the request shape the ticket reports keeps working on system-first templates; only a closing run that follows an assistant turn stays in place * refactor(openai): reuse is_openai_backed_api_base for the hosted endpoint check * test(base_utils): cover an empty system message inside a block-content run Codecov flagged the empty-string branch of _text_blocks as the one line the PR left uncovered. The existing empty-string test only hits the plain-text join; this one forces the block path by merging an empty system message with a developer message that carries list content. * fix(openai): hoist developer messages for litellm_proxy and harden the system fold - drop the LiteLLMProxyChatConfig passthrough so litellm_proxy/ hoists later developer messages like every other OpenAI-compatible endpoint, whichever backend or LiteLLM version sits downstream - a system message with null or missing content inside a folded run is dropped instead of raising KeyError('content') - a message-level cache_control on a list-content member lands on its last block so the breakpoint survives the fold * fix(litellm_proxy): keep the AllMessageValues import the transform signatures still name * fix(fireworks_ai): hoist developer items after the first turn on the native Responses path The native Fireworks Responses path landed on staging mapping developer input items to system in place, so a developer item after the first input item still hit the same "System message must be at the beginning" 400 on system-first chat templates. The ordering rules now live in one shared helper over item roles, and the native path reorders through it before mapping, keeping a closing developer item after an assistant turn in place exactly like the chat path does * fix(transformation): carry the developer-message hoist onto main 963333d654 Resolves the main merge for this branch: the tests move to tests/unit, every translate_developer_role_to_system_role override takes and returns a Sequence, Azure OpenAI keeps the in-place mapping, and the system content reads tolerate a missing key * fix(transformation): join a lone plain text block as text when hoisting developer messages The Responses bridge turns every developer item into a one-block list, so the hoisted run merged the instructions string and that block as a block list, and Databricks could no longer collapse it into the single system string it sends today. A lone text block with no extra keys now joins the run as text, while multi-block, cached, and billing-header content keep the block path * fix(transformation): hoist later developer messages only for models that reject mid-conversation system messages Runs of consecutive system messages still fold into one, which keeps the cached prefix stable. A developer message after the first turn moves into the leading system message only when the model declares supports_mid_conversation_system false, or when it is a Fireworks model with no declaration, matching the native Fireworks Responses route. Every other backend keeps it in place as before, so their prompt cache prefix is unchanged. Malformed system content passes through unmerged so the provider returns its own 400. * fix(responses): fold later developer items into instructions on hosted_vllm for system-first models Moves the Responses developer fold shared by Fireworks into one module and applies it to hosted_vllm native Responses when the model declares supports_mid_conversation_system false. Renames the run-merging helper so it no longer shares a name with the Databricks one, and returns tuples from the block helpers. * fix(llms): place developer messages only for models declared system-first Models without a supports_mid_conversation_system declaration now keep the in-place developer-to-system translation they had before, so their cached prompt prefix does not change. Hoisting and folding run only when the deployment's model_info or the cost map declares the flag false, with the deployment value read first. A developer message that closes the conversation after an assistant turn is sent as a user message on those models instead of being hoisted, merged list content keeps a blank-line separator before each later part, and the cost map declares the flag false for fireworks_ai qwen3p8-max * fix(proxy): keep a closing developer message's cache mark when it is sent as a user turn * test: type the developer-hoist tests and fake HTTP with respx * fix: count non-string roles as turns, keep the Fireworks fold for twins, and copy closing developer content A message whose role is a list or dict now counts as a conversation turn instead of raising a 500. Fireworks native Responses skips the developer-item fold only when the deployment's own model_info declares supports_mid_conversation_system true, since the router registers every deployment's flag under its shared backend key. A hoisted developer message keeps only role, content, name, and cache_control, a closing developer message's content is deep-copied so the caller's list is left alone, and a hosted_vllm developer item with no content is forwarded unchanged. The Fireworks in-place test now declares true on the deployment itself, because a cost-map true can no longer be told apart from another deployment's registration. --------- Co-authored-by: sowmy narayanan Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/llms/anthropic/chat/transformation.py | 5 +- .../messages/native/transformation.py | 5 +- litellm/llms/azure/chat/gpt_transformation.py | 19 +- litellm/llms/base_llm/base_utils.py | 238 ++++++++- litellm/llms/base_llm/chat/transformation.py | 15 +- .../fireworks_ai/responses/transformation.py | 128 +---- .../hosted_vllm/responses/transformation.py | 28 + .../llms/openai/chat/gpt_transformation.py | 75 ++- .../openai/chat/o_series_transformation.py | 12 +- litellm/llms/openai/openai.py | 21 +- .../openai/responses/instruction_items.py | 98 ++++ litellm/main.py | 17 +- ...odel_prices_and_context_window_backup.json | 2 + litellm/utils.py | 37 ++ model_prices_and_context_window.json | 2 + tests/unit/conftest.py | 14 + .../test_azure_chat_gpt_transformation.py | 23 + .../chat/test_azure_ai_transformation.py | 23 + tests/unit/llms/base_llm/test_base_utils.py | 494 ++++++++++++++++++ .../test_fireworks_ai_chat_transformation.py | 84 +++ ...t_fireworks_ai_responses_transformation.py | 102 +++- .../responses/test_hosted_vllm_responses.py | 93 ++++ .../test_litellm_proxy_chat_transformation.py | 22 + .../chat/test_openai_gpt_transformation.py | 96 ++++ .../openai/test_o_series_transformation.py | 17 + tests/unit/llms/openai/test_openai.py | 40 +- tests/unit/test_utils.py | 38 ++ 28 files changed, 1612 insertions(+), 137 deletions(-) create mode 100644 litellm/llms/openai/responses/instruction_items.py create mode 100644 tests/unit/llms/base_llm/test_base_utils.py diff --git a/litellm/constants.py b/litellm/constants.py index 7f2d98b7a28..d02114f56b1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -627,6 +627,7 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096 diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..865247afc63 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -12,6 +12,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_BILLING_METADATA_PREFIX, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS, ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES, DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS, @@ -2066,7 +2067,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if not system_message_block["content"]: continue if self.should_strip_billing_metadata() and system_message_block["content"].startswith( - "x-anthropic-billing-header:" + ANTHROPIC_BILLING_METADATA_PREFIX ): continue anthropic_system_message_content = AnthropicSystemMessageContent( @@ -2086,7 +2087,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): self.should_strip_billing_metadata() and _content.get("type") == "text" and text_value - and text_value.startswith("x-anthropic-billing-header:") + and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX) ): continue anthropic_system_message_content = AnthropicSystemMessageContent( diff --git a/litellm/llms/anthropic/messages/native/transformation.py b/litellm/llms/anthropic/messages/native/transformation.py index 6c0e463bd23..75b0329315f 100644 --- a/litellm/llms/anthropic/messages/native/transformation.py +++ b/litellm/llms/anthropic/messages/native/transformation.py @@ -6,6 +6,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -207,7 +208,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): """ if isinstance(system_param, str): # If it's a string and starts with billing header, filter it out - if system_param.startswith("x-anthropic-billing-header:"): + if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): return None return system_param elif isinstance(system_param, list): @@ -218,7 +219,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): text = content_block.get("text", "") content_type = content_block.get("type", "") # Skip text blocks that start with billing header - if content_type == "text" and text.startswith("x-anthropic-billing-header:"): + if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): continue filtered_list.append(content_block) else: diff --git a/litellm/llms/azure/chat/gpt_transformation.py b/litellm/llms/azure/chat/gpt_transformation.py index 26ae30635d4..990c8869de3 100644 --- a/litellm/llms/azure/chat/gpt_transformation.py +++ b/litellm/llms/azure/chat/gpt_transformation.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( convert_to_azure_openai_messages, ) +from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_reasoning_transformation import GPT_REASONING_SERIES_MARKERS from litellm.types.llms.azure import ( @@ -98,6 +99,22 @@ class AzureOpenAIConfig(BaseConfig): if key != "self" and value is not None: setattr(self.__class__, key, value) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + Azure OpenAI accepts `system` messages at any position, so a developer message + only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it + ahead of the OpenAI-compatible hoist it also inherits. + """ + return map_developer_role_to_system_role(messages=messages) + @classmethod def get_config(cls): return super().get_config() diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index a9ec1f97d20..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -10,14 +10,23 @@ from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final +from typing import Any, Final, TypeAlias from openai.lib import _parsing, _pydantic # pyright: ignore[reportPrivateUsage] # SDK parser internals from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError +from typing_extensions import TypeIs # noqa: TID251 # narrows a message to its system member by the role literal from litellm._logging import verbose_logger +from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX from litellm.litellm_core_utils.core_helpers import as_str_mapping -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.llms.openai import ( + AllMessageValues, + ChatCompletionCachedContent, + ChatCompletionSystemMessage, + ChatCompletionTextObject, + ChatCompletionToolCallChunk, + ChatCompletionUserMessage, +) from litellm.types.utils import ( CountContextManagement, Message, @@ -458,7 +467,7 @@ def type_to_response_format_param( def map_developer_role_to_system_role( - messages: list[AllMessageValues], + messages: Sequence[AllMessageValues], ) -> list[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. @@ -473,3 +482,226 @@ def map_developer_role_to_system_role( else: new_messages.append(m) return new_messages + + +SystemMessageContent: TypeAlias = str | list[object] + +_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None) + + +def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent | None: + if "content" not in message: + return "" + try: + content: Final = _system_content_adapter.validate_python(message["content"]) + except ValidationError: + return None + return "" if content is None else content + + +def _message_cache_control(message: ChatCompletionSystemMessage) -> ChatCompletionCachedContent | None: + if "cache_control" not in message: + return None + return message["cache_control"] + + +_SystemPart: TypeAlias = tuple[SystemMessageContent, ChatCompletionCachedContent | None] + + +def _with_message_cache_control( + blocks: Sequence[object], cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + last: Final = as_str_mapping(blocks[-1]) if blocks else None + if last is None or cache_control is None or "cache_control" in last: + return tuple(blocks) + return (*blocks[:-1], {**last, "cache_control": cache_control}) + + +def _as_system_message(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached + + +def _text_blocks( + content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None +) -> tuple[object, ...]: + if not isinstance(content, str): + return _with_message_cache_control(content, cache_control) + if not content: + return () + if cache_control: + cached_block: Final[ChatCompletionTextObject] = { + "type": "text", + "text": content, + "cache_control": cache_control, + } + return (cached_block,) + text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content} + return (text_block,) + + +def _lone_text_block_text(blocks: Sequence[object]) -> str | None: + match blocks: + case [{"type": "text", "text": str() as text} as block] if block == {"type": "text", "text": text}: + return text + case _: + return None + + +def _plain_text(content: SystemMessageContent, cache_control: ChatCompletionCachedContent | None) -> str | None: + if cache_control: + return None + text: Final = content if isinstance(content, str) else _lone_text_block_text(content) + if text is None or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return None + return text + + +def _is_system_message( + message: AllMessageValues, +) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member + return message["role"] == "system" + + +def _with_leading_separator(blocks: tuple[object, ...]) -> tuple[object, ...]: + first: Final = as_str_mapping(blocks[0]) + text: Final = None if first is None else first.get("text") + if first is None or not isinstance(text, str) or text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX): + return blocks + return ({**first, "text": f"\n\n{text}"}, *blocks[1:]) + + +def _merged_system_content(parts: Sequence[_SystemPart]) -> SystemMessageContent: + texts: Final = tuple(itertools.starmap(_plain_text, parts)) + if all(text is not None for text in texts): + return "\n\n".join(text for text in texts if text) + block_runs: Final = tuple(blocks for blocks in itertools.starmap(_text_blocks, parts) if blocks) + separated: Final = (*block_runs[:1], *map(_with_leading_separator, block_runs[1:])) + return list(itertools.chain.from_iterable(separated)) + + +def _merged_system_message( + run: Sequence[ChatCompletionSystemMessage], parts: Sequence[_SystemPart] +) -> ChatCompletionSystemMessage: + content: Final = _merged_system_content(parts) + named: Final = next((message for message in reversed(run) if "name" in message), None) + if named is None: + merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content} + return merged + with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]} + return with_name + + +def _merged_run(run: tuple[AllMessageValues, ...]) -> tuple[AllMessageValues, ...]: + system_run: Final = tuple(message for message in run if _is_system_message(message)) + if len(system_run) < 2: + return run + contents: Final = tuple((_system_content(message), _message_cache_control(message)) for message in system_run) + parts: Final = tuple((content, cache_control) for content, cache_control in contents if content is not None) + if len(parts) != len(contents): + return run + return (_merged_system_message(system_run, parts),) + + +def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]: + runs: Final = (tuple(run) for _, run in itertools.groupby(messages, key=_is_system_message)) + return itertools.chain.from_iterable(map(_merged_run, runs)) + + +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Indices of the leading instruction block plus every developer item after it, except + a developer block that closes the conversation right after an assistant turn, which + stays in place so the request does not end on the assistant's turn. Items without a + role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer") + return (*range(leading_length), *moved) + + +def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: + closing_start: Final = _closing_instruction_block_start(roles, _leading_instruction_block_length(roles)) + return frozenset(range(closing_start, len(roles))) + + +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + +def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) + hoisted_set: Final = frozenset(hoisted) + return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) + + +def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: + if message["role"] != "developer": + return message + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} + return as_user + cached_as_user: Final[ChatCompletionUserMessage] = { + "role": "user", + "content": copied["content"], + "cache_control": copied["cache_control"], + } + return cached_as_user + + +def merge_system_message_runs(messages: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + """ + Translate `developer` role to `system` role in place and fold each run of + consecutive system messages into one, for chat templates that render a single + system message per run. + """ + return tuple(_merged_system_runs(tuple(map(_as_system_message, messages)))) + + +def hoist_developer_messages_into_leading_system_message( + messages: Sequence[AllMessageValues], +) -> Sequence[AllMessageValues]: + """ + Place instructions for an OpenAI-compatible model whose chat template allows a single + system message and only at the start: developer messages after the first user turn move + into the leading system block, a developer message that closes the conversation right + after an assistant turn becomes a user turn so the request does not end on the + assistant's turn, and each run of consecutive system messages folds into one. + """ + order: Final = _hoisted_message_order(messages) + closing: Final = closing_instruction_indices(_message_roles(messages)) + placed: Final = tuple( + _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order + ) + return merge_system_message_runs(placed) diff --git a/litellm/llms/base_llm/chat/transformation.py b/litellm/llms/base_llm/chat/transformation.py index 207a4705c92..b87a783e244 100644 --- a/litellm/llms/base_llm/chat/transformation.py +++ b/litellm/llms/base_llm/chat/transformation.py @@ -4,7 +4,7 @@ Common base config for all LLM providers import types from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Iterator, Mapping +from collections.abc import AsyncIterator, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, Union import httpx @@ -199,12 +199,19 @@ class BaseConfig(ABC): def translate_developer_role_to_system_role( self, - messages: list[AllMessageValues], - ) -> list[AllMessageValues]: + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ Translate `developer` role to `system` role for non-OpenAI providers. - Overriden by OpenAI/Azure + Overridden by OpenAI, which also places the instructions in one leading system + message for an OpenAI-compatible model declared to reject a later system + message, and by the o-series and Azure configs. """ return map_developer_role_to_system_role(messages=messages) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index db234c09930..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,10 +1,9 @@ -from collections.abc import Mapping, Sequence +from collections.abc import Mapping from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( @@ -15,12 +14,18 @@ from litellm.llms.fireworks_ai.common_utils import ( with_fireworks_session_affinity, without_caller_user, ) +from litellm.llms.openai.responses.instruction_items import developer_items_as_system, instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -45,90 +50,6 @@ def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, ) -_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) - - -def _role(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": str(role)}: - return role - case _: - return None - - -def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": - return item - return EasyInputMessageParam(role="system", content=item["content"], type="message") - - -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: - if isinstance(input, str): - return input - return [_developer_item_as_system(item) for item in input] - - -def _text_part(part: ResponseInputContentParam) -> str | None: - match part: - case {"type": "input_text", "text": str(text)}: - return text - case _: - return None - - -def _text_only_content(item: ResponseInputItemParam) -> str | None: - match item: - case {"role": "system" | "developer", "content": str(text)}: - return text - case {"role": "system" | "developer", "content": [*parts]}: - texts: Final = tuple(map(_text_part, parts)) - return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) - case _: - return None - - -def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: - return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) - - -def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: - last_conversation_index: Final = next( - (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), - None, - ) - if last_conversation_index is None or roles[last_conversation_index] != "assistant": - return len(roles) - return last_conversation_index + 1 - - -def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: - leading_length: Final = _leading_instruction_block_length(roles) - closing_start: Final = _closing_instruction_block_start(roles, leading_length) - return tuple( - index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" - ) - - -def _with_instruction_items_folded( - input: str | ResponseInputParam, instructions: str | None -) -> tuple[str | None, str | ResponseInputParam]: - if isinstance(input, str): - return instructions, input - items: Final = tuple(input) - folded: Final = MappingProxyType( - { - index: text - for index in _hoisted_indices(tuple(map(_role, items))) - if (text := _text_only_content(items[index])) is not None - } - ) - joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) - return ( - instructions if not folded else joined or None, - [_developer_item_as_system(item) for index, item in enumerate(items) if index not in folded], - ) - - class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @property def custom_llm_provider(self) -> LlmProviders: @@ -158,29 +79,28 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): self, model: str, input: str | ResponseInputParam, - response_api_optional_request_params: dict, # mutable-ok: overrides the base class signature + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature litellm_params: GenericLiteLLMParams, - headers: dict, # mutable-ok: overrides the base class signature - ) -> dict: # mutable-ok: overrides the base class signature - instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - instructions, folded_input = ( - _with_instruction_items_folded(validated_input, instructions_param) - if isinstance(instructions_param, str | None) - else (instructions_param, _developer_items_as_system(validated_input)) - ) - instruction_entries: Final = () if instructions is None else (("instructions", instructions),) - folded_params: Final = { - key: value - for key, value in ( - *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), - *instruction_entries, + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) + folded: Final = ( + None + if deployment_declared_mid_conversation_system_support(model_info) is True + else instruction_items_folded( + validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) - } + ) + placed_input, placed_params = folded or ( + developer_items_as_system(validated_input), + response_api_optional_request_params, + ) request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=folded_input, - response_api_optional_request_params=folded_params, + input=placed_input, + response_api_optional_request_params=dict(placed_params), litellm_params=litellm_params, headers=headers, ) diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 8489bc0da84..0831ed3b37a 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -9,10 +9,13 @@ the chat completions → responses conversion pipeline. from typing import Final from litellm.litellm_core_utils.url_utils import join_api_base +from litellm.llms.openai.responses.instruction_items import instruction_items_folded from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ResponseInputParam from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders +from litellm.utils import declared_mid_conversation_system_support, deployment_model_info class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,6 +72,31 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): return join_api_base(api_base, "v1/responses") + def transform_responses_api_request( + self, + model: str, + input: str | ResponseInputParam, + response_api_optional_request_params: dict[str, object], # mutable-ok: overrides the base class signature + litellm_params: GenericLiteLLMParams, + headers: dict[str, str], # mutable-ok: overrides the base class signature + ) -> dict[str, object]: # mutable-ok: overrides the base class signature + declared: Final = declared_mid_conversation_system_support( + model, self.custom_llm_provider.value, deployment_model_info(litellm_params) + ) + folded: Final = ( + instruction_items_folded(self._validate_input_param(input), response_api_optional_request_params, "user") + if declared is False + else None + ) + placed_input, placed_params = folded or (input, response_api_optional_request_params) + return super().transform_responses_api_request( + model=model, + input=placed_input, + response_api_optional_request_params=dict(placed_params), + litellm_params=litellm_params, + headers=headers, + ) + def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" return False diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index b7396e8fb03..22af51aaabe 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -36,7 +36,11 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( ) from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator -from litellm.llms.base_llm.base_utils import BaseLLMModelInfo +from litellm.llms.base_llm.base_utils import ( + BaseLLMModelInfo, + hoist_developer_messages_into_leading_system_message, + map_developer_role_to_system_role, +) from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -58,7 +62,7 @@ from litellm.types.utils import ( ModelResponseStream, chat_completion_tool_call_from_dict, ) -from litellm.utils import convert_to_model_response_object +from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support from ..common_utils import OpenAIError from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config @@ -75,6 +79,45 @@ else: _NO_TOOLS_UPDATE: Final[Mapping[str, object]] = MappingProxyType({}) + + +def targets_openai_hosted_endpoint(custom_llm_provider: str | None, api_base: str | None) -> bool: + if custom_llm_provider != "openai": + return False + resolved_api_base: Final = ( + api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") + ) + if not resolved_api_base: + return True + hostname: Final = urlparse(resolved_api_base).hostname + if hostname is None: + return True + return hostname == "openai.com" or hostname.endswith(".openai.com") + + +def _roles(messages: Sequence[AllMessageValues]) -> tuple[str, ...]: + return tuple(message["role"] for message in messages) + + +def translate_developer_role_for_openai_compatible_endpoint( + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, +) -> Sequence[AllMessageValues]: + in_place: Final = map_developer_role_to_system_role(messages=messages) + if targets_openai_hosted_endpoint(custom_llm_provider, api_base): + return in_place + placed: Final = hoist_developer_messages_into_leading_system_message(messages=messages) + if _roles(placed) == _roles(in_place): + return in_place + if declared_mid_conversation_system_support(model, custom_llm_provider, model_info) is not False: + return in_place + return placed + + _OBJECT_MAPPING: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) _CHOICE_MAPPINGS: Final = TypeAdapter(tuple[Mapping[str, object], ...]) _DECODED_JSON: Final = TypeAdapter(object) @@ -437,20 +480,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): ) return messages, tools + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + """ + A developer message only changes role, except on a model whose deployment or cost + map declares `supports_mid_conversation_system` false, where the instructions are + placed in one leading system message for its chat template. + """ + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _targets_openai_hosted_endpoint( self, custom_llm_provider: str | None, api_base: str | None, ) -> bool: - if custom_llm_provider != "openai": - return False - resolved_api_base = api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE") - if not resolved_api_base: - return True - hostname: Final = urlparse(resolved_api_base).hostname - if hostname is None: - return True - return hostname == "openai.com" or hostname.endswith(".openai.com") + return targets_openai_hosted_endpoint(custom_llm_provider, api_base) def _should_preserve_cache_control_for_endpoint( self, diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index c7552d47985..64285aafdb0 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -11,7 +11,7 @@ Translations handled by LiteLLM: - Logprobs => drop param (if user opts in to dropping param) """ -from collections.abc import Coroutine +from collections.abc import Coroutine, Sequence from typing import Final, Literal, cast, overload import litellm @@ -37,7 +37,15 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): def get_config(cls): return super().get_config() - def translate_developer_role_to_system_role(self, messages: list[AllMessageValues]) -> list[AllMessageValues]: + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: """ O-series models support `developer` role. """ diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 5a3ab8aebea..af2e85c5f20 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1,6 +1,6 @@ import time import types -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from typing import TYPE_CHECKING, Final, Literal, Optional, cast import httpx @@ -50,7 +50,11 @@ from litellm.utils import ( from ...types.llms.openai import * from ..base import BaseLLM from .chat.gpt_reasoning_transformation import OpenAIGPT5Config -from .chat.gpt_transformation import OpenAIGPTConfig, OpenAIUnknownModelConfig +from .chat.gpt_transformation import ( + OpenAIGPTConfig, + OpenAIUnknownModelConfig, + translate_developer_role_for_openai_compatible_endpoint, +) from .chat.o_series_transformation import OpenAIOSeriesConfig from .common_utils import ( BaseOpenAILLM, @@ -206,6 +210,19 @@ class OpenAIConfig(BaseConfig): else: return self._gpt_config_for_model(model).get_supported_openai_params(model=model) + def translate_developer_role_to_system_role( + self, + messages: Sequence[AllMessageValues], + *, + model: str, + custom_llm_provider: str | None, + api_base: str | None, + model_info: object, + ) -> Sequence[AllMessageValues]: + return translate_developer_role_for_openai_compatible_endpoint( + messages, model=model, custom_llm_provider=custom_llm_provider, api_base=api_base, model_info=model_info + ) + def _gpt_config_for_model(self, model: str) -> OpenAIGPTConfig: if type(self) is OpenAIConfig and not OpenAIGPTConfig.is_openai_catalog_model(model): return OpenAIUnknownModelConfig() diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py new file mode 100644 index 00000000000..82bf4329b34 --- /dev/null +++ b/litellm/llms/openai/responses/instruction_items.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam + +from litellm.llms.base_llm.base_utils import hoisted_instruction_indices +from litellm.types.llms.openai import ResponseInputParam + +UnfoldedDeveloperRole: TypeAlias = Literal["system", "user"] + +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) + + +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None + + +def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer" or "content" not in item: + return item + return EasyInputMessageParam(role=role, content=item["content"], type="message") + + +def developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): + return input + return [_developer_item_as(item, "system") for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _folded_instructions_and_items( + items: tuple[ResponseInputItemParam, ...], instructions: str | None, unfolded_developer_role: UnfoldedDeveloperRole +) -> tuple[str | None, ResponseInputParam]: + folded: Final = MappingProxyType( + { + index: text + for index in hoisted_instruction_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [_developer_item_as(item, unfolded_developer_role) for index, item in enumerate(items) if index not in folded], + ) + + +def instruction_items_folded( + input: str | ResponseInputParam, + params: Mapping[str, object], + unfolded_developer_role: UnfoldedDeveloperRole, +) -> tuple[ResponseInputParam, Mapping[str, object]] | None: + """ + For a model whose chat template takes a single leading system message, the + leading system and developer items and every later text developer item folded into + top-level `instructions`, the developer items left over sent with + `unfolded_developer_role`, or None when the request needs no folding. + """ + instructions: Final = params.get("instructions") + if isinstance(input, str) or not isinstance(instructions, str | None): + return None + items: Final = tuple(input) + if not any(_role(item) in _INSTRUCTION_ROLES for item in items): + return None + folded_instructions, folded_items = _folded_instructions_and_items(items, instructions, unfolded_developer_role) + instruction_entries: Final = () if folded_instructions is None else (("instructions", folded_instructions),) + return folded_items, MappingProxyType( + { + key: value + for key, value in ( + *((key, value) for key, value in params.items() if key != "instructions"), + *instruction_entries, + ) + } + ) diff --git a/litellm/main.py b/litellm/main.py index 996e93589fe..a3a63d5a83a 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5703,15 +5703,16 @@ def completion( ), ) + deployment_model_info: Final = str_keyed_mapping( + cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg + ) ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name responses_api_model_info, model = responses_api_bridge_check( model=model, custom_llm_provider=custom_llm_provider, web_search_options=web_search_options, api_base=api_base, - deployment_model_info=str_keyed_mapping( - cast(object, model_info) # cast-ok: model_info is the untyped dict kwarg - ), + deployment_model_info=deployment_model_info, ) routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict drop_unsupported_reasoning: Final = ( @@ -5808,7 +5809,15 @@ def completion( ) if provider_config is not None: - translated_messages: Final = provider_config.translate_developer_role_to_system_role(messages=messages) + translated_messages: Final = list( + provider_config.translate_developer_role_to_system_role( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=deployment_model_info, + ) + ) history_for_route: Final = _history_without_unanswered_tool_search_calls( translated_messages, custom_llm_provider, provider_config ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 50b3560f05b..aa00ea73f8c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/litellm/utils.py b/litellm/utils.py index aeb25f81841..adf195668ee 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -220,6 +220,7 @@ def _get_cached_audio_utils(): from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper +from litellm.types.llms.base_model import LiteLLMBaseModel from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -3501,6 +3502,42 @@ def supports_mid_conversation_system(model: str, custom_llm_provider: str | None ) +class _DeploymentModelInfo(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + model_info: Mapping[str, object] | None = None + + +def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None: + """The deployment's own ``model_info`` read off its litellm params.""" + return _DeploymentModelInfo.model_validate(litellm_params).model_info + + +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: + match model_info: + case {"supports_mid_conversation_system": bool(declared)}: + return declared + case _: + return None + + +def declared_mid_conversation_system_support( + model: str, custom_llm_provider: str | None, model_info: object +) -> bool | None: + """ + The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the + cost map declares for the model, else None when neither sets it. + """ + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) + if deployment_declared is not None: + return deployment_declared + if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): + return False + if supports_mid_conversation_system(model, custom_llm_provider): + return True + return None + + def supports_native_structured_output(model: str, custom_llm_provider: str | None = None) -> bool: """ Check if the given model supports native structured outputs and return a boolean value. diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 50b3560f05b..aa00ea73f8c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -63655,6 +63655,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -63801,6 +63802,7 @@ "output_cost_per_token_priority": 9e-06, "source": "https://api.fireworks.ai/v1/serverless/models", "supports_function_calling": true, + "supports_mid_conversation_system": false, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 3070ebb0235..ed7dd5ec146 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -378,6 +378,20 @@ def local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: litellm.get_model_info.cache_clear() +@pytest.fixture +def declare_mid_conversation_system(monkeypatch: pytest.MonkeyPatch) -> Callable[[str, bool], None]: + def declare(model_cost_key: str, supported: bool) -> None: + entry: Final = { + "litellm_provider": model_cost_key.split("/", 1)[0], + "mode": "chat", + "supports_mid_conversation_system": supported, + } + monkeypatch.setitem(litellm.model_cost, model_cost_key, entry) + litellm.get_model_info.cache_clear() + + return declare + + NativeStructuredOutputKey: TypeAlias = Literal["legacy", "bedrock"] | None diff --git a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py index 53df6610913..f6dae2c304f 100644 --- a/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py +++ b/tests/unit/llms/azure/chat/test_azure_chat_gpt_transformation.py @@ -783,3 +783,26 @@ def test_responses_api_version_leaves_the_chat_route_and_body_untouched(respx_mo assert request.url.params["api-version"] == "2024-12-01-preview" assert json.loads(request.content) == {"model": "gpt-5.4-nano", "messages": messages} assert response.choices[0].message.content == "Hello" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("config", [AzureOpenAIConfig(), AzureOpenAIGPT5Config()]) +def test_azure_keeps_a_later_developer_message_in_place_as_system(config: AzureOpenAIConfig) -> None: + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="my-deployment", + custom_llm_provider="azure", + api_base="https://example-resource.openai.azure.com", + model_info=None, + ) + assert [message["role"] for message in messages] == ["system", "user", "assistant", "system", "user"] + assert messages[3]["content"] == "Answer with exactly one word." diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index ccac6ffc309..42dc222e420 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -1,5 +1,6 @@ import json import traceback +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -1098,3 +1099,25 @@ def test_map_azure_model_group(model_group_header, expected_model): config = AzureAICohereConfig() assert config._map_azure_model_group(model_group_header) == expected_model + + +def test_translate_developer_role_hoists_a_later_developer_message_for_a_model_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("azure_ai/system-first-model", False) + messages: Final = AzureAIStudioConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="system-first-model", + custom_llm_provider="azure_ai", + api_base="https://example.services.ai.azure.com/models", + model_info=None, + ) + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "user"] + assert messages[0]["content"] == "You are terse.\n\nAnswer with exactly one word." diff --git a/tests/unit/llms/base_llm/test_base_utils.py b/tests/unit/llms/base_llm/test_base_utils.py new file mode 100644 index 00000000000..7052fd04242 --- /dev/null +++ b/tests/unit/llms/base_llm/test_base_utils.py @@ -0,0 +1,494 @@ +from typing import Final + +from litellm.llms.base_llm.base_utils import ( + hoist_developer_messages_into_leading_system_message, + merge_system_message_runs, +) + + +class TestMergeSystemMessageRuns: + def test_later_developer_message_stays_in_place_while_the_leading_run_folds(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Sandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex\n\nSandbox: read-only."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_run_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + assert list(merge_system_message_runs(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": {"text": "not a content list"}}, + {"role": "user", "content": "Hi there"}, + ] + + def test_hoisted_developer_message_with_content_that_is_neither_text_nor_blocks_is_forwarded_unmerged(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": 42}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": 42}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +class TestHoistDeveloperMessagesIntoLeadingSystemMessage: + def test_single_developer_message(self) -> None: + messages: Final = [ + {"role": "developer", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_developer_messages_merge(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Rule 1: Be concise."}, + {"role": "developer", "content": "Rule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rule 1: Be concise.\n\nRule 2: Respond in Markdown."}, + {"role": "user", "content": "Hello"}, + ] + + def test_system_followed_by_developer_merges(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base instructions."}, + {"role": "developer", "content": "Developer override."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base instructions.\n\nDeveloper override."}, + {"role": "user", "content": "Hello"}, + ] + + def test_developer_followed_by_system_merges(self) -> None: + messages: Final = [ + {"role": "developer", "content": "Developer instruction."}, + {"role": "system", "content": "System instruction."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Developer instruction.\n\nSystem instruction."}, + {"role": "user", "content": "Hello"}, + ] + + def test_consecutive_native_system_messages_merge(self) -> None: + messages: Final = [ + {"role": "system", "content": "System part 1."}, + {"role": "system", "content": "System part 2."}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System part 1.\n\nSystem part 2."}, + {"role": "user", "content": "Hello"}, + ] + + def test_empty_string_side_does_not_pad(self) -> None: + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": ""}, {"role": "developer", "content": "Rules"}] + ) + ) == [{"role": "system", "content": "Rules"}] + assert list( + hoist_developer_messages_into_leading_system_message( + [{"role": "system", "content": "Rules"}, {"role": "developer", "content": ""}] + ) + ) == [{"role": "system", "content": "Rules"}] + + def test_empty_string_message_adds_no_block_when_the_run_merges_as_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": ""}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "Hi"}, + ] + + def test_two_cached_string_messages_keep_one_breakpoint_each(self) -> None: + messages: Final = [ + {"role": "system", "content": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"role": "system", "content": "B", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A", "cache_control": {"type": "ephemeral", "ttl": "5m"}}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + ], + }, + ] + + def test_cache_control_supplied_only_by_the_later_message_marks_only_its_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "A"}, + {"role": "developer", "content": "B", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "A"}, + {"type": "text", "text": "\n\nB", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + + def test_cache_control_of_the_first_message_stays_on_its_own_block(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "developer", "content": "Additional instructions"}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAdditional instructions"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_cached_string_message_merged_with_block_content_keeps_its_breakpoint(self) -> None: + messages: Final = [ + {"role": "system", "content": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": [{"type": "text", "text": "Answer with exactly one word."}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached prompt", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_plain_messages_ahead_of_a_cached_one_keep_their_own_blocks(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "developer", "content": "Rule"}, + {"role": "developer", "content": "Cached", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Base"}, + {"type": "text", "text": "\n\nRule"}, + {"type": "text", "text": "\n\nCached", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "Hello"}, + ] + + def test_name_of_the_last_named_message_survives_a_long_run(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base", "name": "first"}, + *({"role": "developer", "content": f"Rule {index}"} for index in range(2000)), + {"role": "developer", "content": "Last", "name": "last"}, + {"role": "user", "content": "Hello"}, + ] + merged: Final = list(hoist_developer_messages_into_leading_system_message(messages)) + assert len(merged) == 2 + assert merged[0]["name"] == "last" + assert merged[0]["content"] == "\n\n".join(["Base", *(f"Rule {index}" for index in range(2000)), "Last"]) + + def test_billing_metadata_keeps_its_own_block_when_merged(self) -> None: + billing: Final = {"role": "system", "content": "x-anthropic-billing-header: cc_version=1.0;"} + advisory: Final = {"role": "system", "content": "Guardrail advisory: request was flagged"} + assert list(hoist_developer_messages_into_leading_system_message([billing, advisory])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + {"type": "text", "text": "\n\nGuardrail advisory: request was flagged"}, + ], + } + ] + assert list(hoist_developer_messages_into_leading_system_message([advisory, billing])) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Guardrail advisory: request was flagged"}, + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1.0;"}, + ], + } + ] + + def test_developer_message_after_a_user_turn_is_hoisted_into_the_leading_system_message(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "developer", "content": "Developer turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "System turn 1\n\nDeveloper turn 2"}, + {"role": "user", "content": "User turn 1"}, + {"role": "user", "content": "User turn 2"}, + ] + + def test_hoisted_developer_message_keeps_only_the_fields_a_system_message_has(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word.", "name": "codex", "extra_key": "x"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word.", "name": "codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_developer_message_in_second_position_without_a_leading_system_message_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_every_later_developer_message_is_hoisted_in_order_across_assistant_and_tool_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "developer", "content": "Update B"}, + {"role": "user", "content": "Turn 3"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A\n\nUpdate B"}, + {"role": "user", "content": "Turn 1"}, + {"role": "assistant", "content": "Reply 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "user", "content": "Turn 3"}, + ] + + def test_developer_message_that_closes_the_conversation_after_an_assistant_turn_becomes_a_user_turn(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with the single word PONG and nothing else."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "user", "content": "Reply with the single word PONG and nothing else."}, + ] + + def test_closing_developer_message_keeps_its_cache_mark_as_a_user_turn(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello! How can I help?"}, + {"role": "developer", "content": "Reply with PONG.", "cache_control": {"type": "ephemeral"}}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages))[-1] == { + "role": "user", + "content": "Reply with PONG.", + "cache_control": {"type": "ephemeral"}, + } + + def test_only_the_closing_developer_run_after_an_assistant_turn_becomes_user_turns(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Turn 1"}, + {"role": "developer", "content": "Update A"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "developer", "content": "Closing B"}, + {"role": "developer", "content": "Closing C"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nUpdate A"}, + {"role": "user", "content": "Turn 1"}, + {"role": "user", "content": "Turn 2"}, + {"role": "assistant", "content": "Reply 2"}, + {"role": "user", "content": "Closing B"}, + {"role": "user", "content": "Closing C"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_user_turn_is_hoisted(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Reply with exactly one word: the capital of France"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Reply with exactly one word: the capital of France"}, + {"role": "user", "content": "Hi there"}, + ] + + def test_developer_message_that_closes_the_conversation_after_a_tool_result_is_hoisted(self) -> None: + messages: Final = [ + {"role": "system", "content": "Base"}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + {"role": "developer", "content": "Answer with exactly one word."}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Base\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Look it up"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "Paris"}, + ] + + def test_hoisted_single_text_block_developer_message_joins_string_instructions_as_text(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "developer", "content": [{"type": "text", "text": ""}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "You are Codex\n\n"}, + {"role": "user", "content": [{"type": "text", "text": "# AGENTS.md"}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the capital of France?"}]}, + ] + + def test_multi_block_developer_message_keeps_its_blocks_after_string_instructions(self) -> None: + messages: Final = [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi"}, + {"role": "developer", "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}]}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Codex"}, + {"type": "text", "text": "\n\nRule 1"}, + {"type": "text", "text": "Rule 2"}, + ], + }, + {"role": "user", "content": "Hi"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_system_message_without_content_folds_into_the_run_instead_of_raising(self) -> None: + messages: Final = [ + {"role": "system", "content": "Rules"}, + {"role": "system"}, + {"role": "system", "content": None}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Rules"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_of_a_block_content_member_lands_on_its_last_block(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1"}, {"type": "text", "text": "Rule 2"}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1"}, + {"type": "text", "text": "Rule 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_message_level_cache_control_does_not_override_a_block_that_already_carries_one(self) -> None: + messages: Final = [ + { + "role": "system", + "content": [{"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}], + "cache_control": {"type": "ephemeral"}, + }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Rule 1", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, + {"type": "text", "text": "\n\nAnswer with exactly one word."}, + ], + }, + {"role": "user", "content": "What is the capital of France?"}, + ] + + def test_non_consecutive_native_system_messages_stay_where_the_client_put_them(self) -> None: + messages: Final = [ + {"role": "system", "content": "System turn 1"}, + {"role": "user", "content": "User turn 1"}, + {"role": "system", "content": "System turn 2"}, + {"role": "user", "content": "User turn 2"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == messages + + def test_list_content_merge_preserves_block_level_cache_control(self) -> None: + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}]}, + {"role": "developer", "content": [{"type": "text", "text": "Extra"}]}, + {"role": "user", "content": "Hello"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + { + "role": "system", + "content": [ + {"type": "text", "text": "Cached", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "\n\nExtra"}, + ], + }, + {"role": "user", "content": "Hello"}, + ] diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 61e5545b8bf..28f4fdead9e 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,4 +1,5 @@ import json +from collections.abc import Callable from typing import Final from unittest.mock import MagicMock, patch @@ -2358,3 +2359,86 @@ def test_global_disable_flag_with_transform_messages_helper( ) assert json.loads(route.calls.last.request.content)["messages"] == [{"role": "user", "content": content}] + + +_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN: Final = ( + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +def test_translate_developer_role_keeps_a_later_developer_message_in_place_for_an_undeclared_fireworks_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +def test_translate_developer_role_leaves_a_leading_system_and_developer_pair_unmerged_for_an_undeclared_model() -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are Codex"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="accounts/fireworks/models/unlisted-model", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize("model", ["qwen3p8-max", "accounts/fireworks/models/qwen3p8-max"]) +def test_translate_developer_role_hoists_a_later_developer_message_for_the_system_first_qwen3p8_model( + local_model_cost_map: None, model: str +) -> None: + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model=model, + custom_llm_provider="fireworks_ai", + api_base=None, + model_info=None, + ) + + assert list(messages) == [ + {"role": "system", "content": "You are Codex\n\nAnswer with exactly one word."}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("deployment_declared", "expected_roles"), + [(False, ["system", "user", "user"]), (True, ["system", "user", "system", "user"])], +) +def test_translate_developer_role_follows_the_deployment_model_info_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], deployment_declared: bool, expected_roles: list[str] +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", not deployment_declared) + messages: Final = FireworksAIConfig().translate_developer_role_to_system_role( + messages=_CODEX_DEVELOPER_MESSAGE_AFTER_A_USER_TURN, + model="accounts/fireworks/models/shared-backend", + custom_llm_provider="fireworks_ai", + api_base=None, + model_info={"id": "deployment-a", "supports_mid_conversation_system": deployment_declared}, + ) + + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 878e0285cd1..7fc6ac2c436 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -1,5 +1,5 @@ import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import MappingProxyType from typing import Final, TypedDict, cast from unittest.mock import MagicMock, patch @@ -7,6 +7,7 @@ from urllib.parse import quote import httpx import pytest +import respx from openai.types.responses import ( EasyInputMessage, ResponseFunctionToolCall, @@ -81,6 +82,14 @@ def _mock_http_client(response_body: Mapping[str, object]) -> MagicMock: return client +def _fireworks_responses_route(model: str) -> respx.Route: + return respx.post(FIREWORKS_RESPONSES_URL).mock(return_value=httpx.Response(200, json=_fireworks_response(model))) + + +def _routed_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + def _sent_request(client: MagicMock) -> tuple[str, Mapping[str, str], Mapping[str, object]]: kwargs: Final = client.post.call_args.kwargs body: Final = kwargs["json"] if "json" in kwargs else json.loads(kwargs["data"]) @@ -182,6 +191,60 @@ def test_responses_call_folds_developer_items_into_instructions() -> None: ) +@respx.mock +def test_responses_call_keeps_developer_items_in_place_for_a_deployment_that_accepts_later_system_items() -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/accepts-later-system") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/accepts-later-system", + instructions="You are Codex", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model_info={"id": "deployment-1", "supports_mid_conversation_system": True}, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "You are Codex" + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, + {"role": "user", "content": "What is the capital of France?"}, + ) + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "closing_role"), + [(None, "system"), ({"id": "deployment-1", "supports_mid_conversation_system": False}, "user")], +) +def test_responses_call_sends_the_closing_developer_item_as_a_user_turn_only_for_a_declared_system_first_deployment( + model_info: Mapping[str, object] | None, closing_role: str +) -> None: + route: Final = _fireworks_responses_route("accounts/fireworks/models/kimi-k3") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": "developer", "content": "Reply with PONG."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body["instructions"] == "Answer with exactly one word." + assert tuple(body["input"]) == ( + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": "Paris"}, + {"role": closing_role, "content": "Reply with PONG.", "type": "message"}, + ) + + def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): @@ -667,3 +730,40 @@ def test_responses_call_forwards_litellm_user_id_as_user( _, _, body = _sent_request(client) assert body.get("user") == expected_user assert "fireworks_forward_user_id" not in body + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "expected_instructions", "expected_input"), + [ + (None, "Answer in one word.", ({"role": "user", "content": "What is the capital of France?"},)), + ( + {"id": "deployment-1", "supports_mid_conversation_system": True}, + None, + ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Answer in one word.", "type": "message"}, + ), + ), + ], +) +def test_responses_call_forwards_developer_items_in_place_only_when_the_deployment_itself_declares_true( + declare_mid_conversation_system: Callable[[str, bool], None], + model_info: Mapping[str, object] | None, + expected_instructions: str | None, + expected_input: tuple[Mapping[str, object], ...], +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", True) + route: Final = _fireworks_responses_route("accounts/fireworks/models/shared-backend") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/shared-backend", + input=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "developer", "content": "Answer in one word."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body.get("instructions") == expected_instructions + assert tuple(body["input"]) == expected_input diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index e32f876a6b9..2f63ef070e6 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -8,9 +8,13 @@ hosted_vllm (and any OpenAI-compatible provider using add_provider_specific_para """ import json +from collections.abc import Callable, Mapping +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm @@ -183,3 +187,92 @@ def test_hosted_vllm_validate_environment_custom_api_key(): ) assert headers.get("Authorization") == "Bearer my-custom-key" + + +_LATER_DEVELOPER_INPUT: Final = ( + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, +) + + +_VLLM_RESPONSES_URL: Final = "https://test-vllm.example.com/v1/responses" + + +def _vllm_responses_route(output_text: str) -> respx.Route: + return respx.post(_VLLM_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_make_mock_responses_api_response(output_text)) + ) + + +def _sent_body(route: respx.Route) -> Mapping[str, object]: + return json.loads(route.calls.last.request.content) + + +@respx.mock +def test_hosted_vllm_responses_folds_a_later_developer_item_into_instructions_for_a_system_first_model( + declare_mid_conversation_system: Callable[[str, bool], None], +) -> None: + declare_mid_conversation_system("hosted_vllm/Qwen/Qwen3.6-27B", False) + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3.6-27B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == (_LATER_DEVELOPER_INPUT[0], _LATER_DEVELOPER_INPUT[2]) + + +@respx.mock +def test_hosted_vllm_responses_folds_for_a_deployment_declared_system_first_and_sends_a_closing_developer_as_user() -> None: + assistant_turn: Final = {"type": "message", "role": "assistant", "content": "Paris"} + route: Final = _vllm_responses_route("PONG") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=[*_LATER_DEVELOPER_INPUT, assistant_turn, {"role": "developer", "content": "Reply with PONG."}], + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == ( + _LATER_DEVELOPER_INPUT[0], + _LATER_DEVELOPER_INPUT[2], + assistant_turn, + {"role": "user", "content": "Reply with PONG.", "type": "message"}, + ) + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_later_developer_item_in_place_for_an_undeclared_model() -> None: + route: Final = _vllm_responses_route("Paris") + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + instructions="You are a terse assistant.", + input=list(_LATER_DEVELOPER_INPUT), + api_base="https://test-vllm.example.com/v1", + ) + body: Final = _sent_body(route) + assert body["instructions"] == "You are a terse assistant." + assert tuple(body["input"]) == _LATER_DEVELOPER_INPUT + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_developer_item_without_content_on_a_deployment_declared_system_first() -> None: + route: Final = _vllm_responses_route("Paris") + input_items: Final = ( + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"type": "message", "role": "developer"}, + ) + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input=list(input_items), + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + assert tuple(_sent_body(route)["input"]) == input_items diff --git a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py index 9b8788d3769..8a690a70a97 100644 --- a/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py +++ b/tests/unit/llms/litellm_proxy/chat/test_litellm_proxy_chat_transformation.py @@ -639,3 +639,25 @@ def test_litellm_gateway_from_sdk_with_thinking_param(respx_mock): ) e = exc_info.value assert "Connection error." in str(e) + + +def test_translate_developer_role_leaves_a_later_developer_message_in_place_for_the_downstream_proxy() -> None: + translated: Final = LiteLLMProxyChatConfig().translate_developer_role_to_system_role( + messages=[ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ], + model="fireworks-qwen", + custom_llm_provider="litellm_proxy", + api_base="http://inner-proxy:4000", + model_info=None, + ) + + assert list(translated) == [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 70436662ef0..5b85a94b24d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,7 +2,9 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ import base64 +import copy import json +from collections.abc import Callable from typing import Final import pytest @@ -1820,3 +1822,97 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: ) assert request["tools"][0] is tool + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("config", "custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + (OpenAIGPTConfig(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPT5Config(), "openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "openai", "http://vllm.internal:8000/v1", False, ["system", "user", "assistant", "user"]), + (OpenAIGPTConfig(), "groq", None, None, ["system", "user", "assistant", "system", "user"]), + (OpenAIGPTConfig(), "groq", None, False, ["system", "user", "assistant", "user"]), + ], +) +def test_translate_developer_role_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + config: OpenAIGPTConfig, + custom_llm_provider: str, + api_base: str | None, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = config.translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles + assert "Answer with exactly one word." in [message["content"] for message in messages if message["role"] == "system"][-1] + + +@pytest.mark.parametrize("role", [["user"], {"name": "user"}]) +@pytest.mark.parametrize( + ("declared", "expected_order"), + [(None, ("caller", "instruction")), (False, ("instruction", "caller"))], +) +def test_translate_developer_role_places_a_message_with_a_non_string_role_like_any_conversation_turn( + declare_mid_conversation_system: Callable[[str, bool], None], + role: object, + declared: bool | None, + expected_order: tuple[str, str], +) -> None: + if declared is not None: + declare_mid_conversation_system("groq/system-first-model", declared) + caller_turn: Final = {"role": role, "content": "Hi there"} + by_name: Final = {"caller": caller_turn, "instruction": {"role": "system", "content": "Answer in one word."}} + messages: Final = OpenAIGPTConfig().translate_developer_role_to_system_role( + messages=[caller_turn, {"role": "developer", "content": "Answer in one word."}], + model="system-first-model", + custom_llm_provider="groq", + api_base=None, + model_info=None, + ) + assert list(messages) == [by_name[name] for name in expected_order] + + +@pytest.mark.respx(assert_all_called=True) +def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_system_first_deployment( + respx_mock: respx.MockRouter, +) -> None: + chat: Final = respx_mock.post("http://vllm.internal:8000/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + messages: Final = [ + {"role": "user", "content": "Describe the image."}, + {"role": "assistant", "content": "Send it."}, + { + "role": "developer", + "content": [{"type": "text", "text": "Look at this."}, {"type": "image_url", "image_url": "data:image/png;base64,aW1hZ2U="}], + }, + ] + sent: Final = copy.deepcopy(messages) + litellm.completion( + model="openai/system-first-model", + api_base="http://vllm.internal:8000/v1", + api_key="sk-test", + messages=messages, + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + ) + assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" + assert messages == sent diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index 121fab66c15..e78e1a6f12b 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -181,3 +181,20 @@ def test_litellm_responses(): print("response: ", response) assert isinstance(response.usage.completion_tokens_details, CompletionTokensDetails) + + +def test_translate_developer_role_keeps_developer_messages_untouched() -> None: + messages: Final = [ + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert ( + OpenAIOSeriesConfig().translate_developer_role_to_system_role( + messages=messages, + model="my-reasoning-deployment", + custom_llm_provider="openai", + api_base=None, + model_info=None, + ) + == messages + ) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 11d7c7d6d3c..5b27a446fb3 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -2,7 +2,7 @@ import asyncio import importlib import json import os -from collections.abc import Mapping +from collections.abc import Callable, Mapping from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncCursorPage, MessageData, OpenAIChatCompletion, + OpenAIConfig, OpenAIMessage as Message, Run, SyncCursorPage, @@ -2010,3 +2011,40 @@ async def test_chat_completion_sends_the_caller_params_extra_body_and_extra_head assert json.loads(sent.content) == {**_WIRE_REQUEST, "vendor_flag": True} assert sent.headers["x-caller"] == "unit" assert sent.headers["authorization"] == "Bearer sk-test" + + +def _developer_message_after_a_user_turn() -> list[dict[str, str]]: + return [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": "Hi there"}, + {"role": "assistant", "content": "Hello!"}, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + ] + + +@pytest.mark.parametrize( + ("custom_llm_provider", "api_base", "declared", "expected_roles"), + [ + ("openai", "https://api.openai.com/v1", False, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", None, ["system", "user", "assistant", "system", "user"]), + ("azure_ai", "https://example.services.ai.azure.com/models", False, ["system", "user", "assistant", "user"]), + ], +) +def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_openai_that_rejects_it( + declare_mid_conversation_system: Callable[[str, bool], None], + custom_llm_provider: str, + api_base: str, + declared: bool | None, + expected_roles: list[str], +) -> None: + if declared is not None: + declare_mid_conversation_system(f"{custom_llm_provider}/system-first-model", declared) + messages: Final = OpenAIConfig().translate_developer_role_to_system_role( + messages=_developer_message_after_a_user_turn(), + model="system-first-model", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + model_info=None, + ) + assert [message["role"] for message in messages] == expected_roles diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 665a540e462..2870016a5e9 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -10869,6 +10869,44 @@ def test_get_optional_params_keeps_a_tool_config_cache_point_out_of_the_openai_e assert "cache_control_injection_points" not in optional_params +@pytest.mark.parametrize( + ("cost_map_declared", "deployment_declared", "expected"), + [ + (None, None, None), + (False, None, False), + (True, None, True), + (True, False, False), + (False, True, True), + (None, False, False), + ], +) +def test_declared_mid_conversation_system_support_prefers_the_deployment_over_the_cost_map( + declare_mid_conversation_system: Callable[[str, bool], None], + cost_map_declared: bool | None, + deployment_declared: bool | None, + expected: bool | None, +) -> None: + if cost_map_declared is not None: + declare_mid_conversation_system("hosted_vllm/mid-conversation-probe", cost_map_declared) + deployment_model_info: Final = ( + {"id": "deployment-1"} + if deployment_declared is None + else {"id": "deployment-1", "supports_mid_conversation_system": deployment_declared} + ) + assert ( + litellm.utils.declared_mid_conversation_system_support( + "mid-conversation-probe", "hosted_vllm", deployment_model_info + ) + is expected + ) + + +def test_declared_mid_conversation_system_support_is_none_when_the_model_lookup_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem(litellm.model_cost, "hosted_vllm/unreadable-probe", "not a model info entry") + litellm.get_model_info.cache_clear() + assert litellm.utils.declared_mid_conversation_system_support("unreadable-probe", "hosted_vllm", None) is None + + def test_get_optional_params_forwards_a_tool_config_cache_point_to_a_downstream_litellm_proxy(): """A litellm_proxy deployment calls the downstream proxy through the OpenAI SDK, and that proxy applies cache_control_injection_points itself, so the point must ride along in extra_body."""