diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index 660a07181ff..f7dd774ea18 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -1,16 +1,10 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final from urllib.parse import unquote import httpx -from openai.types.responses import ( - EasyInputMessageParam, - ResponseInputContentParam, - ResponseInputItemParam, - ResponseInputTextParam, -) -from pydantic import TypeAdapter +from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.llms.fireworks_ai.common_utils import ( resolve_fireworks_api_key, @@ -37,47 +31,90 @@ def _session_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object ) -_instructions_adapter: Final = TypeAdapter[str | None](str | None) +_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) -def _instruction_parts(item: ResponseInputItemParam) -> tuple[ResponseInputContentParam, ...] | None: - if "role" not in item or (item["role"] != "system" and item["role"] != "developer"): - return None - content: Final = item["content"] - if isinstance(content, str): - return (ResponseInputTextParam(type="input_text", text=content),) - return tuple(content) +def _role(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": str(role)}: + return role + case _: + return None -def _leading_system_content( - instructions: str | None, parts: tuple[ResponseInputContentParam, ...] -) -> str | list[ResponseInputContentParam]: - text: Final = "\n\n".join( - chunk for chunk in (instructions, *(part["text"] for part in parts if part["type"] == "input_text")) if chunk - ) - non_text: Final = tuple(part for part in parts if part["type"] != "input_text") - if not non_text: - return text - return [ResponseInputTextParam(type="input_text", text=text), *non_text] if text else list(non_text) +def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam: + if "role" not in item or item["role"] != "developer": + return item + return EasyInputMessageParam(role="system", content=item["content"], type="message") -def _with_single_leading_system_item( - input: str | ResponseInputParam, instructions: str | None -) -> str | ResponseInputParam: - items: Final = () if isinstance(input, str) else tuple(input) - instruction_parts: Final = tuple( - part for item_parts in map(_instruction_parts, items) if item_parts is not None for part in item_parts - ) - content: Final = _leading_system_content(instructions, instruction_parts) - if not content: +def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam: + if isinstance(input, str): return input - leading: Final = EasyInputMessageParam(role="system", content=content, type="message") - rest: Final = ( - (EasyInputMessageParam(role="user", content=input),) - if isinstance(input, str) - else tuple(item for item in items if _instruction_parts(item) is None) + return [_developer_item_as_system(item) for item in input] + + +def _text_part(part: ResponseInputContentParam) -> str | None: + match part: + case {"type": "input_text", "text": str(text)}: + return text + case _: + return None + + +def _text_only_content(item: ResponseInputItemParam) -> str | None: + match item: + case {"role": "system" | "developer", "content": str(text)}: + return text + case {"role": "system" | "developer", "content": [*parts]}: + texts: Final = tuple(map(_text_part, parts)) + return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text) + case _: + return None + + +def _leading_instruction_block_length(roles: Sequence[str | None]) -> int: + return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles)) + + +def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int: + last_conversation_index: Final = next( + (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES), + None, + ) + if last_conversation_index is None or roles[last_conversation_index] != "assistant": + return len(roles) + return last_conversation_index + 1 + + +def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]: + leading_length: Final = _leading_instruction_block_length(roles) + closing_start: Final = _closing_instruction_block_start(roles, leading_length) + return tuple( + index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer" + ) + + +def _with_instruction_items_folded( + input: str | ResponseInputParam, instructions: str | None +) -> tuple[str | None, str | ResponseInputParam]: + if isinstance(input, str): + return instructions, input + items: Final = tuple(input) + folded: Final = MappingProxyType( + { + index: text + for index in _hoisted_indices(tuple(map(_role, items))) + if (text := _text_only_content(items[index])) is not None + } + ) + joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk) + return ( + instructions if not folded else joined or None, + [ # mutable-ok: the base class takes the input items as a list + _developer_item_as_system(item) for index, item in enumerate(items) if index not in folded + ], ) - return [leading, *rest] class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): @@ -113,15 +150,24 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: overrides the base class signature ) -> dict: # mutable-ok: overrides the base class signature - instructions: Final = _instructions_adapter.validate_python( - response_api_optional_request_params.get("instructions") + instructions_param: Final[object] = response_api_optional_request_params.get("instructions") + validated_input: Final = self._validate_input_param(input) + instructions, folded_input = ( + _with_instruction_items_folded(validated_input, instructions_param) + if isinstance(instructions_param, str | None) + else (instructions_param, _developer_items_as_system(validated_input)) ) + instruction_entries: Final = () if instructions is None else (("instructions", instructions),) folded_params: Final = { # mutable-ok: the base class takes the optional params as a dict - key: value for key, value in response_api_optional_request_params.items() if key != "instructions" + key: value + for key, value in ( + *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"), + *instruction_entries, + ) } return super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), - input=_with_single_leading_system_item(self._validate_input_param(input), instructions), + input=folded_input, response_api_optional_request_params=folded_params, litellm_params=litellm_params, headers=headers, diff --git a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 9d947b25d69..d0697ca9b0e 100644 --- a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -162,7 +162,7 @@ def test_responses_call_forwards_previous_response_id_and_store() -> None: assert body["input"][0]["call_id"] == "call_abc123" -def test_responses_call_hoists_developer_items_into_one_leading_system_message() -> None: +def test_responses_call_folds_developer_items_into_instructions() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): litellm.responses( @@ -175,14 +175,14 @@ def test_responses_call_hoists_developer_items_into_one_leading_system_message() api_key="fw-test-key", ) _, _, body = _sent_request(client) + assert body["instructions"] == "Answer with exactly one word." assert tuple(body["input"]) == ( - {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, {"role": "user", "content": "Hi there"}, {"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]}, ) -def test_responses_call_folds_instructions_and_developer_item_into_one_leading_system_message() -> None: +def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): litellm.responses( @@ -208,16 +208,10 @@ def test_responses_call_folds_instructions_and_developer_item_into_one_leading_s api_key="fw-test-key", ) _, _, body = _sent_request(client) - assert "instructions" not in body + assert body["instructions"] == ( + "You are a coding agent running in the Codex CLI.\n\nread-only" + ) assert tuple(body["input"]) == ( - { - "role": "system", - "content": ( - "You are a coding agent running in the Codex CLI.\n\n" - "read-only" - ), - "type": "message", - }, {"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]}, {"id": "rs_1", "type": "reasoning", "summary": [{"type": "summary_text", "text": "A trivial question."}]}, { @@ -231,41 +225,103 @@ def test_responses_call_folds_instructions_and_developer_item_into_one_leading_s ) -def test_responses_call_keeps_non_text_developer_parts_on_the_leading_system_message() -> None: +def test_responses_call_folds_instructions_and_developer_item_with_previous_response_id() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): litellm.responses( model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b", - instructions="Answer with one word.", + instructions="You are a terse assistant.", input=[ # mutable-ok: the Responses API takes input as a JSON list - { - "role": "developer", - "content": [ - {"type": "input_text", "text": "Match the style of this reference image."}, - {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, - ], - }, + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "And of Spain?"}, + ], + previous_response_id="resp_0e946f2d46bf4b49bf8b29ff78083583", + store=True, + api_key="fw-test-key", + ) + _, _, body = _sent_request(client) + assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word." + assert body["previous_response_id"] == "resp_0e946f2d46bf4b49bf8b29ff78083583" + assert tuple(body["input"]) == ({"role": "user", "content": "And of Spain?"},) + + +def test_responses_call_keeps_a_closing_developer_item_after_an_assistant_turn_in_place() -> None: + client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) + assistant_turn: Final = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "Paris.", "annotations": []}], + } + with patch(HTTPX_CLIENT_FACTORY, return_value=client): + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b", + instructions="Be terse.", + input=[ # mutable-ok: the Responses API takes input as a JSON list + {"role": "developer", "content": "Answer with exactly one word."}, + {"role": "user", "content": "What is the capital of France?"}, + assistant_turn, + {"role": "developer", "content": "Now restate it in French."}, + ], + api_key="fw-test-key", + ) + _, _, body = _sent_request(client) + assert body["instructions"] == "Be terse.\n\nAnswer with exactly one word." + assert tuple(body["input"]) == ( + {"role": "user", "content": "What is the capital of France?"}, + assistant_turn, + {"role": "system", "content": "Now restate it in French.", "type": "message"}, + ) + + +def test_responses_call_keeps_a_mid_conversation_system_item_in_place() -> None: + client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) + with patch(HTTPX_CLIENT_FACTORY, return_value=client): + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + input=[ # mutable-ok: the Responses API takes input as a JSON list + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Switch to French."}, {"role": "user", "content": "What is the capital of France?"}, ], - store=False, api_key="fw-test-key", ) _, _, body = _sent_request(client) assert "instructions" not in body assert tuple(body["input"]) == ( - { - "role": "system", - "content": [ - {"type": "input_text", "text": "Answer with one word.\n\nMatch the style of this reference image."}, - {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, - ], - "type": "message", - }, + {"role": "user", "content": "Hi there"}, + {"role": "system", "content": "Switch to French."}, {"role": "user", "content": "What is the capital of France?"}, ) -def test_responses_call_turns_string_input_with_instructions_into_system_then_user_messages() -> None: +def test_responses_call_keeps_a_developer_item_with_non_text_parts_in_place_as_a_system_item() -> None: + client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b")) + developer_item: Final = { + "role": "developer", + "content": [ + {"type": "input_text", "text": "Match the style of this reference image."}, + {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"}, + ], + } + with patch(HTTPX_CLIENT_FACTORY, return_value=client): + litellm.responses( + model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b", + instructions="Answer with one word.", + input=[developer_item, {"role": "user", "content": "What is the capital of France?"}], # mutable-ok: JSON list + store=False, + api_key="fw-test-key", + ) + _, _, body = _sent_request(client) + assert body["instructions"] == "Answer with one word." + assert tuple(body["input"]) == ( + {"role": "system", "content": developer_item["content"], "type": "message"}, + {"role": "user", "content": "What is the capital of France?"}, + ) + + +def test_responses_call_forwards_string_input_and_instructions_unchanged() -> None: client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) with patch(HTTPX_CLIENT_FACTORY, return_value=client): litellm.responses( @@ -275,10 +331,24 @@ def test_responses_call_turns_string_input_with_instructions_into_system_then_us api_key="fw-test-key", ) _, _, body = _sent_request(client) - assert "instructions" not in body - assert tuple(body["input"]) == ( + assert body["instructions"] == "Answer with exactly one word." + assert body["input"] == "What is the capital of France?" + + +def test_transform_request_forwards_non_string_instructions_and_input_untouched() -> None: + developer_item: Final = {"role": "developer", "content": "Answer with exactly one word."} + user_item: Final = {"role": "user", "content": "What is the capital of France?"} + request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request( + model="accounts/fireworks/models/kimi-k3", + input=cast(ResponseInputParam, [developer_item, user_item]), # mutable-ok: JSON list + response_api_optional_request_params={"instructions": ["not", "a", "string"]}, # mutable-ok: base takes a dict + litellm_params=GenericLiteLLMParams(), + headers={}, # mutable-ok: base takes a dict + ) + assert request["instructions"] == ["not", "a", "string"] + assert tuple(request["input"]) == ( {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, - {"role": "user", "content": "What is the capital of France?"}, + user_item, ) @@ -305,8 +375,8 @@ def test_responses_call_maps_pydantic_developer_items_and_replays_pydantic_outpu model="fireworks_ai/accounts/fireworks/models/kimi-k3", input=pydantic_input, api_key="fw-test-key" ) _, _, body = _sent_request(client) + assert body["instructions"] == "Answer with exactly one word." assert tuple(body["input"]) == ( - {"role": "system", "content": "Answer with exactly one word.", "type": "message"}, {"id": "rs_1", "summary": [], "type": "reasoning"}, { "id": "fc_1",