diff --git a/litellm/llms/base_llm/base_utils.py b/litellm/llms/base_llm/base_utils.py index 5a6f2507dda..1193dc068b8 100644 --- a/litellm/llms/base_llm/base_utils.py +++ b/litellm/llms/base_llm/base_utils.py @@ -321,45 +321,46 @@ def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMes _INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) -def _leading_system_block_length(messages: Sequence[AllMessageValues]) -> int: - return next( - (index for index, message in enumerate(messages) if message["role"] not in _INSTRUCTION_ROLES), - len(messages), - ) +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) -def _closing_instruction_block_start(messages: Sequence[AllMessageValues], leading_length: int) -> int: +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: last_conversation_index: Final = next( - ( - index - for index in range(len(messages) - 1, leading_length - 1, -1) - if messages[index]["role"] not in _INSTRUCTION_ROLES - ), + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), None, ) - if last_conversation_index is None or messages[last_conversation_index]["role"] != "assistant": - return len(messages) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) return last_conversation_index + 1 -def _move_later_developer_messages_up(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: - leading_length: Final = _leading_system_block_length(messages) - closing_start: Final = _closing_instruction_block_start(messages, leading_length) - conversation: Final = messages[leading_length:closing_start] - hoisted: Final = tuple(message for message in conversation if message["role"] == "developer") +def hoisted_developer_item_order(roles: Sequence[str | None]) -> tuple[int, ...]: + """ + Index order that moves every developer item after the leading instruction block + up into it, except a developer block that closes the conversation right after an + assistant turn, which stays in place so the request does not end on the assistant's + turn. Items without a role (tool calls and their outputs) count as conversation. + """ + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + conversation: Final = range(leading_length, closing_start) + hoisted: Final = tuple(index for index in conversation if roles[index] == "developer") if hoisted: - verbose_logger.debug( - "Hoisting %d developer message(s) into the leading system block for OpenAI-compatible backends.", - len(hoisted), - ) + verbose_logger.debug("Hoisting %d developer message(s) into the leading system block.", len(hoisted)) return ( - *messages[:leading_length], + *range(leading_length), *hoisted, - *(message for message in conversation if message["role"] != "developer"), - *messages[closing_start:], + *(index for index in conversation if roles[index] != "developer"), + *range(closing_start, len(roles)), ) +def _move_later_developer_messages_up(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]: + order: Final = hoisted_developer_item_order(tuple(message["role"] for message in messages)) + return tuple(messages[index] for index in order) + + def hoist_developer_messages_into_leading_system_message( messages: Sequence[AllMessageValues], ) -> Sequence[AllMessageValues]: diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index fb0587553d4..b03407f7c26 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -6,6 +6,7 @@ from urllib.parse import unquote import httpx from openai.types.responses import EasyInputMessageParam, ResponseInputItemParam +from litellm.llms.base_llm.base_utils import hoisted_developer_item_order from litellm.llms.fireworks_ai.common_utils import ( resolve_fireworks_api_key, resolve_fireworks_resource_name, @@ -31,16 +32,23 @@ def _session_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object ) +def _item_role(item: ResponseInputItemParam) -> str | None: + if "role" not in item: + return None + return item["role"] + + def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: if "role" not in item or item["role"] != "developer": return item return EasyInputMessageParam(role="system", content=item["content"], type="message") -def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: +def _hoisted_developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: if isinstance(input, str): return input - return [_developer_item_as_system(item) for item in input] + order: Final = hoisted_developer_item_order(tuple(_item_role(item) for item in input)) + return [_developer_item_as_system(input[index]) for index in order] class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -69,7 +77,7 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return f"{base}/responses" def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam: - return _developer_items_as_system(super()._validate_input_param(input)) + return _hoisted_developer_items_as_system(super()._validate_input_param(input)) def transform_responses_api_request( self, diff --git a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index b9408d44e9a..f9733aac14c 100644 --- a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -162,12 +162,13 @@ def test_responses_call_forwards_previous_response_id_and_store() -> None: assert body["input"][0]["call_id"] == "call_abc123" -def test_responses_call_sends_developer_items_as_system_messages() -> None: +def test_responses_call_hoists_developer_items_after_the_first_turn_into_leading_system_messages() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): litellm.responses( model="fireworks_ai/accounts/fireworks/models/kimi-k3", input=[ # mutable-ok: the Responses API takes input as a JSON list + {"role": "system", "content": "You are terse."}, {"role": "user", "content": "Hi there"}, {"role": "developer", "content": "Answer with exactly one word."}, {"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]}, @@ -176,12 +177,33 @@ def test_responses_call_sends_developer_items_as_system_messages() -> None: ) _, _, body = _sent_request(client) assert tuple(body["input"]) == ( - {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "You are terse."}, {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, + {"role": "user", "content": "Hi there"}, {"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]}, ) +def test_responses_call_keeps_a_closing_developer_item_after_an_assistant_turn_in_place() -> None: + client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) + with patch(HTTPX_CLIENT_FACTORY, return_value=client): + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ # mutable-ok: the Responses API takes input as a JSON list + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "Paris.", "annotations": []}]}, + {"role": "developer", "content": "Now answer in French."}, + ], + api_key="fw-test-key", + ) + _, _, body = _sent_request(client) + assert tuple(body["input"]) == ( + {"role": "user", "content": "What is the capital of France?"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "Paris.", "annotations": []}]}, + {"role": "system", "content": "Now answer in French.", "type": "message"}, + ) + + def test_responses_call_maps_pydantic_developer_items_and_replays_pydantic_output_items() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) pydantic_input: Final = cast(