diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 4beb7306ec4..ef8d080027f 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -520,8 +520,14 @@ def _with_message_cache_control( def _as_system_message(message: AllMessageValues) -> AllMessageValues: if message["role"] != "developer": return message - translated: Final[ChatCompletionSystemMessage] = {**message, "role": "system"} - return translated + content_only: Final[ChatCompletionSystemMessage] = {"role": "system", "content": message["content"]} + named: Final[ChatCompletionSystemMessage] = ( + content_only if "name" not in message else {**content_only, "name": message["name"]} + ) + if "cache_control" not in message: + return named + cached: Final[ChatCompletionSystemMessage] = {**named, "cache_control": message["cache_control"]} + return cached def _text_blocks( @@ -645,8 +651,16 @@ def closing_instruction_indices(roles: Sequence[str | None]) -> frozenset[int]: return frozenset(range(closing_start, len(roles))) +def _role_name(role: object) -> str | None: + return role if isinstance(role, str) else None + + +def _message_roles(messages: Sequence[AllMessageValues]) -> tuple[str | None, ...]: + return tuple(_role_name(message["role"]) for message in messages) + + def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, ...]: - hoisted: Final = hoisted_instruction_indices(tuple(message["role"] for message in messages)) + hoisted: Final = hoisted_instruction_indices(_message_roles(messages)) hoisted_set: Final = frozenset(hoisted) return (*hoisted, *(index for index in range(len(messages)) if index not in hoisted_set)) @@ -654,13 +668,14 @@ def _hoisted_message_order(messages: Sequence[AllMessageValues]) -> tuple[int, . def _developer_message_as_user(message: AllMessageValues) -> AllMessageValues: if message["role"] != "developer": return message - if "cache_control" not in message: - as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": message["content"]} + copied: Final = copy.deepcopy(message) + if "cache_control" not in copied: + as_user: Final[ChatCompletionUserMessage] = {"role": "user", "content": copied["content"]} return as_user cached_as_user: Final[ChatCompletionUserMessage] = { "role": "user", - "content": message["content"], - "cache_control": message["cache_control"], + "content": copied["content"], + "cache_control": copied["cache_control"], } return cached_as_user @@ -685,7 +700,7 @@ def hoist_developer_messages_into_leading_system_message( assistant's turn, and each run of consecutive system messages folds into one. """ order: Final = _hoisted_message_order(messages) - closing: Final = closing_instruction_indices(tuple(message["role"] for message in messages)) + closing: Final = closing_instruction_indices(_message_roles(messages)) placed: Final = tuple( _developer_message_as_user(messages[index]) if index in closing else messages[index] for index in order ) diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index 9fd0dfe5a60..86d9df9618a 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -21,7 +21,11 @@ from litellm.types.llms.openai import ResponseInputParam from litellm.types.responses.main import DeleteResponseResult from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from litellm.utils import declared_mid_conversation_system_support, deployment_model_info +from litellm.utils import ( + declared_mid_conversation_system_support, + deployment_declared_mid_conversation_system_support, + deployment_model_info, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -80,12 +84,11 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): headers: dict[str, str], # mutable-ok: overrides the base class signature ) -> dict[str, object]: # mutable-ok: overrides the base class signature validated_input: Final = self._validate_input_param(input) - declared: Final = declared_mid_conversation_system_support( - model, self.custom_llm_provider.value, deployment_model_info(litellm_params) - ) + model_info: Final = deployment_model_info(litellm_params) + declared: Final = declared_mid_conversation_system_support(model, self.custom_llm_provider.value, model_info) folded: Final = ( None - if declared is True + if deployment_declared_mid_conversation_system_support(model_info) is True else instruction_items_folded( validated_input, response_api_optional_request_params, "user" if declared is False else "system" ) diff --git a/litellm/llms/openai/responses/instruction_items.py b/litellm/llms/openai/responses/instruction_items.py index 8d99ec602c8..82bf4329b34 100644 --- a/litellm/llms/openai/responses/instruction_items.py +++ b/litellm/llms/openai/responses/instruction_items.py @@ -21,7 +21,7 @@ def _role(item: ResponseInputItemParam) -> str | None: def _developer_item_as(item: ResponseInputItemParam, role: UnfoldedDeveloperRole) -> ResponseInputItemParam: - if "role" not in item or item["role"] != "developer": + if "role" not in item or item["role"] != "developer" or "content" not in item: return item return EasyInputMessageParam(role=role, content=item["content"], type="message") diff --git a/litellm/utils.py b/litellm/utils.py index 94cc6010f0a..4c4943c5627 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3395,7 +3395,7 @@ def deployment_model_info(litellm_params: object) -> Mapping[str, object] | None return _DeploymentModelInfo.model_validate(litellm_params).model_info -def _deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: +def deployment_declared_mid_conversation_system_support(model_info: object) -> bool | None: match model_info: case {"supports_mid_conversation_system": bool(declared)}: return declared @@ -3410,7 +3410,7 @@ def declared_mid_conversation_system_support( The `supports_mid_conversation_system` the deployment's own model_info declares, else the one the cost map declares for the model, else None when neither sets it. """ - deployment_declared: Final = _deployment_declared_mid_conversation_system_support(model_info) + deployment_declared: Final = deployment_declared_mid_conversation_system_support(model_info) if deployment_declared is not None: return deployment_declared if is_explicitly_disabled_factory(model, custom_llm_provider, "supports_mid_conversation_system"): diff --git a/tests/unit/llms/base_llm/test_base_utils.py b/tests/unit/llms/base_llm/test_base_utils.py index 79177a0e80c..7052fd04242 100644 --- a/tests/unit/llms/base_llm/test_base_utils.py +++ b/tests/unit/llms/base_llm/test_base_utils.py @@ -259,6 +259,18 @@ class TestHoistDeveloperMessagesIntoLeadingSystemMessage: {"role": "user", "content": "User turn 2"}, ] + def test_hoisted_developer_message_keeps_only_the_fields_a_system_message_has(self) -> None: + messages: Final = [ + {"role": "user", "content": "Hi there"}, + {"role": "developer", "content": "Answer with exactly one word.", "name": "codex", "extra_key": "x"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + assert list(hoist_developer_messages_into_leading_system_message(messages)) == [ + {"role": "system", "content": "Answer with exactly one word.", "name": "codex"}, + {"role": "user", "content": "Hi there"}, + {"role": "user", "content": "What is the capital of France?"}, + ] + def test_developer_message_in_second_position_without_a_leading_system_message_is_hoisted(self) -> None: messages: Final = [ {"role": "user", "content": "Hi there"}, diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index d0a65d5b897..7fc6ac2c436 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -192,10 +192,7 @@ def test_responses_call_folds_developer_items_into_instructions() -> None: @respx.mock -def test_responses_call_keeps_developer_items_in_place_for_a_model_that_accepts_later_system_items( - declare_mid_conversation_system: Callable[[str, bool], None], -) -> None: - declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/accepts-later-system", True) +def test_responses_call_keeps_developer_items_in_place_for_a_deployment_that_accepts_later_system_items() -> None: route: Final = _fireworks_responses_route("accounts/fireworks/models/accepts-later-system") litellm.responses( model="fireworks_ai/accounts/fireworks/models/accepts-later-system", @@ -205,6 +202,7 @@ def test_responses_call_keeps_developer_items_in_place_for_a_model_that_accepts_ {"role": "developer", "content": "Answer with exactly one word."}, {"role": "user", "content": "What is the capital of France?"}, ], + model_info={"id": "deployment-1", "supports_mid_conversation_system": True}, api_key="fw-test-key", ) body: Final = _routed_body(route) @@ -732,3 +730,40 @@ def test_responses_call_forwards_litellm_user_id_as_user( _, _, body = _sent_request(client) assert body.get("user") == expected_user assert "fireworks_forward_user_id" not in body + + +@respx.mock +@pytest.mark.parametrize( + ("model_info", "expected_instructions", "expected_input"), + [ + (None, "Answer in one word.", ({"role": "user", "content": "What is the capital of France?"},)), + ( + {"id": "deployment-1", "supports_mid_conversation_system": True}, + None, + ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "system", "content": "Answer in one word.", "type": "message"}, + ), + ), + ], +) +def test_responses_call_forwards_developer_items_in_place_only_when_the_deployment_itself_declares_true( + declare_mid_conversation_system: Callable[[str, bool], None], + model_info: Mapping[str, object] | None, + expected_instructions: str | None, + expected_input: tuple[Mapping[str, object], ...], +) -> None: + declare_mid_conversation_system("fireworks_ai/accounts/fireworks/models/shared-backend", True) + route: Final = _fireworks_responses_route("accounts/fireworks/models/shared-backend") + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/shared-backend", + input=[ + {"role": "user", "content": "What is the capital of France?"}, + {"role": "developer", "content": "Answer in one word."}, + ], + model_info=model_info, + api_key="fw-test-key", + ) + body: Final = _routed_body(route) + assert body.get("instructions") == expected_instructions + assert tuple(body["input"]) == expected_input diff --git a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py index 3efe69e55d5..2f63ef070e6 100644 --- a/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py +++ b/tests/unit/llms/hosted_vllm/responses/test_hosted_vllm_responses.py @@ -259,3 +259,20 @@ def test_hosted_vllm_responses_forwards_a_later_developer_item_in_place_for_an_u body: Final = _sent_body(route) assert body["instructions"] == "You are a terse assistant." assert tuple(body["input"]) == _LATER_DEVELOPER_INPUT + + +@respx.mock +def test_hosted_vllm_responses_forwards_a_developer_item_without_content_on_a_deployment_declared_system_first() -> None: + route: Final = _vllm_responses_route("Paris") + input_items: Final = ( + {"role": "user", "content": "What is the capital of France?"}, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "Paris."}]}, + {"type": "message", "role": "developer"}, + ) + litellm.responses( + model="hosted_vllm/Qwen/Qwen3-8B", + input=list(input_items), + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + api_base="https://test-vllm.example.com/v1", + ) + assert tuple(_sent_body(route)["input"]) == input_items diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 60b44d2d351..5b85a94b24d 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -2,6 +2,7 @@ Tests for OpenAI GPT transformation (litellm/llms/openai/chat/gpt_transformation.py) """ import base64 +import copy import json from collections.abc import Callable from typing import Final @@ -1863,3 +1864,55 @@ def test_translate_developer_role_hoists_a_later_developer_message_only_for_a_mo ) assert [message["role"] for message in messages] == expected_roles assert "Answer with exactly one word." in [message["content"] for message in messages if message["role"] == "system"][-1] + + +@pytest.mark.parametrize("role", [["user"], {"name": "user"}]) +@pytest.mark.parametrize( + ("declared", "expected_order"), + [(None, ("caller", "instruction")), (False, ("instruction", "caller"))], +) +def test_translate_developer_role_places_a_message_with_a_non_string_role_like_any_conversation_turn( + declare_mid_conversation_system: Callable[[str, bool], None], + role: object, + declared: bool | None, + expected_order: tuple[str, str], +) -> None: + if declared is not None: + declare_mid_conversation_system("groq/system-first-model", declared) + caller_turn: Final = {"role": role, "content": "Hi there"} + by_name: Final = {"caller": caller_turn, "instruction": {"role": "system", "content": "Answer in one word."}} + messages: Final = OpenAIGPTConfig().translate_developer_role_to_system_role( + messages=[caller_turn, {"role": "developer", "content": "Answer in one word."}], + model="system-first-model", + custom_llm_provider="groq", + api_base=None, + model_info=None, + ) + assert list(messages) == [by_name[name] for name in expected_order] + + +@pytest.mark.respx(assert_all_called=True) +def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_system_first_deployment( + respx_mock: respx.MockRouter, +) -> None: + chat: Final = respx_mock.post("http://vllm.internal:8000/v1/chat/completions").mock( + return_value=Response(200, json=_OPENAI_CHAT_REPLY) + ) + messages: Final = [ + {"role": "user", "content": "Describe the image."}, + {"role": "assistant", "content": "Send it."}, + { + "role": "developer", + "content": [{"type": "text", "text": "Look at this."}, {"type": "image_url", "image_url": "data:image/png;base64,aW1hZ2U="}], + }, + ] + sent: Final = copy.deepcopy(messages) + litellm.completion( + model="openai/system-first-model", + api_base="http://vllm.internal:8000/v1", + api_key="sk-test", + messages=messages, + model_info={"id": "deployment-1", "supports_mid_conversation_system": False}, + ) + assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" + assert messages == sent