From da2e1ff6190d68873a5c3af173ef71fe7dce51c9 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 12:12:34 -0700
Subject: [PATCH] fix(fireworks): fold system and developer items into
instructions on the responses path
Fireworks renders a Responses request through a chat template that only
accepts a system message at the very beginning, so a request carrying
`instructions`, a developer item, and a replayed reasoning item (the shape
Codex CLI sends from its second prompt on) came back 400 with "System
message must be at the beginning".
The leading system or developer items, and any developer item later in
the conversation, now fold their text into top-level `instructions`,
joined with blank lines, and leave `input`. A developer item that closes
the conversation right after an assistant turn stays where it is as a
system item, as does any system or developer item with an image or file
part, so those parts still reach Fireworks. Mid-conversation system items
stay untouched. Non-string `instructions` pass through unchanged.
Folding into `instructions` rather than a leading system item keeps
`previous_response_id` chaining working, since Fireworks prepends the
stored history to `input` and a leading system item would land after it.
This supersedes the leading system item approach from deaadc21d3 and
4807630c2a on this branch. The leading and closing block rules match the
chat path change in #39852.
---
.../fireworks_ai/responses/transformation.py | 136 +++++++++++------
...t_fireworks_ai_responses_transformation.py | 140 +++++++++++++-----
2 files changed, 196 insertions(+), 80 deletions(-)
diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py
index 660a07181ff..f7dd774ea18 100644
--- a/litellm/llms/fireworks_ai/responses/transformation.py
+++ b/litellm/llms/fireworks_ai/responses/transformation.py
@@ -1,16 +1,10 @@
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from urllib.parse import unquote
import httpx
-from openai.types.responses import (
- EasyInputMessageParam,
- ResponseInputContentParam,
- ResponseInputItemParam,
- ResponseInputTextParam,
-)
-from pydantic import TypeAdapter
+from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam
from litellm.llms.fireworks_ai.common_utils import (
resolve_fireworks_api_key,
@@ -37,47 +31,90 @@ def _session_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object
)
-_instructions_adapter: Final = TypeAdapter[str | None](str | None)
+_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"})
-def _instruction_parts(item: ResponseInputItemParam) -> tuple[ResponseInputContentParam, ...] | None:
- if "role" not in item or (item["role"] != "system" and item["role"] != "developer"):
- return None
- content: Final = item["content"]
- if isinstance(content, str):
- return (ResponseInputTextParam(type="input_text", text=content),)
- return tuple(content)
+def _role(item: ResponseInputItemParam) -> str | None:
+ match item:
+ case {"role": str(role)}:
+ return role
+ case _:
+ return None
-def _leading_system_content(
- instructions: str | None, parts: tuple[ResponseInputContentParam, ...]
-) -> str | list[ResponseInputContentParam]:
- text: Final = "\n\n".join(
- chunk for chunk in (instructions, *(part["text"] for part in parts if part["type"] == "input_text")) if chunk
- )
- non_text: Final = tuple(part for part in parts if part["type"] != "input_text")
- if not non_text:
- return text
- return [ResponseInputTextParam(type="input_text", text=text), *non_text] if text else list(non_text)
+def _developer_item_as_system(item: ResponseInputItemParam) -> ResponseInputItemParam:
+ if "role" not in item or item["role"] != "developer":
+ return item
+ return EasyInputMessageParam(role="system", content=item["content"], type="message")
-def _with_single_leading_system_item(
- input: str | ResponseInputParam, instructions: str | None
-) -> str | ResponseInputParam:
- items: Final = () if isinstance(input, str) else tuple(input)
- instruction_parts: Final = tuple(
- part for item_parts in map(_instruction_parts, items) if item_parts is not None for part in item_parts
- )
- content: Final = _leading_system_content(instructions, instruction_parts)
- if not content:
+def _developer_items_as_system(input: str | ResponseInputParam) -> str | ResponseInputParam:
+ if isinstance(input, str):
return input
- leading: Final = EasyInputMessageParam(role="system", content=content, type="message")
- rest: Final = (
- (EasyInputMessageParam(role="user", content=input),)
- if isinstance(input, str)
- else tuple(item for item in items if _instruction_parts(item) is None)
+ return [_developer_item_as_system(item) for item in input]
+
+
+def _text_part(part: ResponseInputContentParam) -> str | None:
+ match part:
+ case {"type": "input_text", "text": str(text)}:
+ return text
+ case _:
+ return None
+
+
+def _text_only_content(item: ResponseInputItemParam) -> str | None:
+ match item:
+ case {"role": "system" | "developer", "content": str(text)}:
+ return text
+ case {"role": "system" | "developer", "content": [*parts]}:
+ texts: Final = tuple(map(_text_part, parts))
+ return None if any(text is None for text in texts) else "\n\n".join(text for text in texts if text)
+ case _:
+ return None
+
+
+def _leading_instruction_block_length(roles: Sequence[str | None]) -> int:
+ return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles))
+
+
+def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int:
+ last_conversation_index: Final = next(
+ (index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES),
+ None,
+ )
+ if last_conversation_index is None or roles[last_conversation_index] != "assistant":
+ return len(roles)
+ return last_conversation_index + 1
+
+
+def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]:
+ leading_length: Final = _leading_instruction_block_length(roles)
+ closing_start: Final = _closing_instruction_block_start(roles, leading_length)
+ return tuple(
+ index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer"
+ )
+
+
+def _with_instruction_items_folded(
+ input: str | ResponseInputParam, instructions: str | None
+) -> tuple[str | None, str | ResponseInputParam]:
+ if isinstance(input, str):
+ return instructions, input
+ items: Final = tuple(input)
+ folded: Final = MappingProxyType(
+ {
+ index: text
+ for index in _hoisted_indices(tuple(map(_role, items)))
+ if (text := _text_only_content(items[index])) is not None
+ }
+ )
+ joined: Final = "\n\n".join(chunk for chunk in (instructions, *folded.values()) if chunk)
+ return (
+ instructions if not folded else joined or None,
+ [ # mutable-ok: the base class takes the input items as a list
+ _developer_item_as_system(item) for index, item in enumerate(items) if index not in folded
+ ],
)
- return [leading, *rest]
class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
@@ -113,15 +150,24 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: overrides the base class signature
) -> dict: # mutable-ok: overrides the base class signature
- instructions: Final = _instructions_adapter.validate_python(
- response_api_optional_request_params.get("instructions")
+ instructions_param: Final[object] = response_api_optional_request_params.get("instructions")
+ validated_input: Final = self._validate_input_param(input)
+ instructions, folded_input = (
+ _with_instruction_items_folded(validated_input, instructions_param)
+ if isinstance(instructions_param, str | None)
+ else (instructions_param, _developer_items_as_system(validated_input))
)
+ instruction_entries: Final = () if instructions is None else (("instructions", instructions),)
folded_params: Final = { # mutable-ok: the base class takes the optional params as a dict
- key: value for key, value in response_api_optional_request_params.items() if key != "instructions"
+ key: value
+ for key, value in (
+ *((key, value) for key, value in response_api_optional_request_params.items() if key != "instructions"),
+ *instruction_entries,
+ )
}
return super().transform_responses_api_request(
model=resolve_fireworks_resource_name(model),
- input=_with_single_leading_system_item(self._validate_input_param(input), instructions),
+ input=folded_input,
response_api_optional_request_params=folded_params,
litellm_params=litellm_params,
headers=headers,
diff --git a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py
index 9d947b25d69..d0697ca9b0e 100644
--- a/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py
+++ b/tests/test_litellm/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py
@@ -162,7 +162,7 @@ def test_responses_call_forwards_previous_response_id_and_store() -> None:
assert body["input"][0]["call_id"] == "call_abc123"
-def test_responses_call_hoists_developer_items_into_one_leading_system_message() -> None:
+def test_responses_call_folds_developer_items_into_instructions() -> None:
client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3"))
with patch(HTTPX_CLIENT_FACTORY, return_value=client):
litellm.responses(
@@ -175,14 +175,14 @@ def test_responses_call_hoists_developer_items_into_one_leading_system_message()
api_key="fw-test-key",
)
_, _, body = _sent_request(client)
+ assert body["instructions"] == "Answer with exactly one word."
assert tuple(body["input"]) == (
- {"role": "system", "content": "Answer with exactly one word.", "type": "message"},
{"role": "user", "content": "Hi there"},
{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]},
)
-def test_responses_call_folds_instructions_and_developer_item_into_one_leading_system_message() -> None:
+def test_responses_call_folds_instructions_and_developer_item_into_instructions_with_reasoning_replayed() -> None:
client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b"))
with patch(HTTPX_CLIENT_FACTORY, return_value=client):
litellm.responses(
@@ -208,16 +208,10 @@ def test_responses_call_folds_instructions_and_developer_item_into_one_leading_s
api_key="fw-test-key",
)
_, _, body = _sent_request(client)
- assert "instructions" not in body
+ assert body["instructions"] == (
+ "You are a coding agent running in the Codex CLI.\n\nread-only"
+ )
assert tuple(body["input"]) == (
- {
- "role": "system",
- "content": (
- "You are a coding agent running in the Codex CLI.\n\n"
- "read-only"
- ),
- "type": "message",
- },
{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}]},
{"id": "rs_1", "type": "reasoning", "summary": [{"type": "summary_text", "text": "A trivial question."}]},
{
@@ -231,41 +225,103 @@ def test_responses_call_folds_instructions_and_developer_item_into_one_leading_s
)
-def test_responses_call_keeps_non_text_developer_parts_on_the_leading_system_message() -> None:
+def test_responses_call_folds_instructions_and_developer_item_with_previous_response_id() -> None:
client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b"))
with patch(HTTPX_CLIENT_FACTORY, return_value=client):
litellm.responses(
model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b",
- instructions="Answer with one word.",
+ instructions="You are a terse assistant.",
input=[ # mutable-ok: the Responses API takes input as a JSON list
- {
- "role": "developer",
- "content": [
- {"type": "input_text", "text": "Match the style of this reference image."},
- {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
- ],
- },
+ {"role": "developer", "content": "Answer with exactly one word."},
+ {"role": "user", "content": "And of Spain?"},
+ ],
+ previous_response_id="resp_0e946f2d46bf4b49bf8b29ff78083583",
+ store=True,
+ api_key="fw-test-key",
+ )
+ _, _, body = _sent_request(client)
+ assert body["instructions"] == "You are a terse assistant.\n\nAnswer with exactly one word."
+ assert body["previous_response_id"] == "resp_0e946f2d46bf4b49bf8b29ff78083583"
+ assert tuple(body["input"]) == ({"role": "user", "content": "And of Spain?"},)
+
+
+def test_responses_call_keeps_a_closing_developer_item_after_an_assistant_turn_in_place() -> None:
+ client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b"))
+ assistant_turn: Final = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "status": "completed",
+ "content": [{"type": "output_text", "text": "Paris.", "annotations": []}],
+ }
+ with patch(HTTPX_CLIENT_FACTORY, return_value=client):
+ litellm.responses(
+ model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b",
+ instructions="Be terse.",
+ input=[ # mutable-ok: the Responses API takes input as a JSON list
+ {"role": "developer", "content": "Answer with exactly one word."},
+ {"role": "user", "content": "What is the capital of France?"},
+ assistant_turn,
+ {"role": "developer", "content": "Now restate it in French."},
+ ],
+ api_key="fw-test-key",
+ )
+ _, _, body = _sent_request(client)
+ assert body["instructions"] == "Be terse.\n\nAnswer with exactly one word."
+ assert tuple(body["input"]) == (
+ {"role": "user", "content": "What is the capital of France?"},
+ assistant_turn,
+ {"role": "system", "content": "Now restate it in French.", "type": "message"},
+ )
+
+
+def test_responses_call_keeps_a_mid_conversation_system_item_in_place() -> None:
+ client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3"))
+ with patch(HTTPX_CLIENT_FACTORY, return_value=client):
+ litellm.responses(
+ model="fireworks_ai/accounts/fireworks/models/kimi-k3",
+ input=[ # mutable-ok: the Responses API takes input as a JSON list
+ {"role": "user", "content": "Hi there"},
+ {"role": "system", "content": "Switch to French."},
{"role": "user", "content": "What is the capital of France?"},
],
- store=False,
api_key="fw-test-key",
)
_, _, body = _sent_request(client)
assert "instructions" not in body
assert tuple(body["input"]) == (
- {
- "role": "system",
- "content": [
- {"type": "input_text", "text": "Answer with one word.\n\nMatch the style of this reference image."},
- {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
- ],
- "type": "message",
- },
+ {"role": "user", "content": "Hi there"},
+ {"role": "system", "content": "Switch to French."},
{"role": "user", "content": "What is the capital of France?"},
)
-def test_responses_call_turns_string_input_with_instructions_into_system_then_user_messages() -> None:
+def test_responses_call_keeps_a_developer_item_with_non_text_parts_in_place_as_a_system_item() -> None:
+ client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/qwen3p8-2p4t-a95b"))
+ developer_item: Final = {
+ "role": "developer",
+ "content": [
+ {"type": "input_text", "text": "Match the style of this reference image."},
+ {"type": "input_image", "image_url": "data:image/png;base64,iVBORw0KGgo=", "detail": "auto"},
+ ],
+ }
+ with patch(HTTPX_CLIENT_FACTORY, return_value=client):
+ litellm.responses(
+ model="fireworks_ai/accounts/fireworks/models/qwen3p8-2p4t-a95b",
+ instructions="Answer with one word.",
+ input=[developer_item, {"role": "user", "content": "What is the capital of France?"}], # mutable-ok: JSON list
+ store=False,
+ api_key="fw-test-key",
+ )
+ _, _, body = _sent_request(client)
+ assert body["instructions"] == "Answer with one word."
+ assert tuple(body["input"]) == (
+ {"role": "system", "content": developer_item["content"], "type": "message"},
+ {"role": "user", "content": "What is the capital of France?"},
+ )
+
+
+def test_responses_call_forwards_string_input_and_instructions_unchanged() -> None:
client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3"))
with patch(HTTPX_CLIENT_FACTORY, return_value=client):
litellm.responses(
@@ -275,10 +331,24 @@ def test_responses_call_turns_string_input_with_instructions_into_system_then_us
api_key="fw-test-key",
)
_, _, body = _sent_request(client)
- assert "instructions" not in body
- assert tuple(body["input"]) == (
+ assert body["instructions"] == "Answer with exactly one word."
+ assert body["input"] == "What is the capital of France?"
+
+
+def test_transform_request_forwards_non_string_instructions_and_input_untouched() -> None:
+ developer_item: Final = {"role": "developer", "content": "Answer with exactly one word."}
+ user_item: Final = {"role": "user", "content": "What is the capital of France?"}
+ request: Final = FireworksAIResponsesAPIConfig().transform_responses_api_request(
+ model="accounts/fireworks/models/kimi-k3",
+ input=cast(ResponseInputParam, [developer_item, user_item]), # mutable-ok: JSON list
+ response_api_optional_request_params={"instructions": ["not", "a", "string"]}, # mutable-ok: base takes a dict
+ litellm_params=GenericLiteLLMParams(),
+ headers={}, # mutable-ok: base takes a dict
+ )
+ assert request["instructions"] == ["not", "a", "string"]
+ assert tuple(request["input"]) == (
{"role": "system", "content": "Answer with exactly one word.", "type": "message"},
- {"role": "user", "content": "What is the capital of France?"},
+ user_item,
)
@@ -305,8 +375,8 @@ def test_responses_call_maps_pydantic_developer_items_and_replays_pydantic_outpu
model="fireworks_ai/accounts/fireworks/models/kimi-k3", input=pydantic_input, api_key="fw-test-key"
)
_, _, body = _sent_request(client)
+ assert body["instructions"] == "Answer with exactly one word."
assert tuple(body["input"]) == (
- {"role": "system", "content": "Answer with exactly one word.", "type": "message"},
{"id": "rs_1", "summary": [], "type": "reasoning"},
{
"id": "fc_1",