From 921bb461deeca1bfb34430d0c6a0ec11832bea77 Mon Sep 17 00:00:00 2001 From: poodle64 Date: Mon, 21 Sep 2026 12:56:32 +1000 Subject: [PATCH 1/4] fix(responses-bridge): fold list-format system content into instructions again convert_chat_completion_messages_to_responses_api only folded a leading system message into `instructions` when its content was a plain str. List-format content ([{"type": "text", "text": "..."}]), how a client attaching cache_control sends a system prompt and how the Anthropic /v1/messages adapter hands one over, fell through to a `role: "system"` input item instead, which some Responses backends reject outright. This is a regression of #21192: that PR added the list-content fold and it was lost in a later merge, leaving only the str branch. Restore it within the leading run of system messages, matching #40269's rule that a system message after the first non-system message stays a positioned input item so byte-identical prompts keep hitting the cache. A leading list whose blocks are not all plain text (e.g. an image) still falls through to an input item unchanged, rather than dropping the non-text block silently. Fixes #42171 Co-Authored-By: Claude Fable 5.1 --- .../transformation.py | 25 ++++- ...responses_transformation_transformation.py | 104 +++++++++++++++++- 2 files changed, 122 insertions(+), 7 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 3b68b58a6d9..c25d7bd9410 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -536,13 +536,30 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_call_id = msg.get("tool_call_id") if role == "system": - # Extract system message as instructions - if isinstance(content, str) and index < leading_system_count: + # Extract system message as instructions, but only within the leading run of + # system messages: a later one stays a positioned input item so its bytes don't + # unsettle the prompt-cache-stable prefix (#40269). + extracted_instructions: str | None = None + if index < leading_system_count: + if isinstance(content, str): + extracted_instructions = content + elif isinstance(content, list) and all( + isinstance(block, str) or (isinstance(block, dict) and block.get("type") == "text") + for block in content + ): + # Every block is plain text (a bare string or a `{"type": "text", ...}` + # dict), the same shape a client attaching cache_control sends; a + # non-text block (an image, say) fails the `all()` above and falls + # through to the input-item branch below instead of losing it silently. + extracted_instructions = " ".join( + block if isinstance(block, str) else block.get("text", "") for block in content + ) + if extracted_instructions is not None: if instructions: # Concatenate multiple system prompts with a space - instructions = f"{instructions} {content}" + instructions = f"{instructions} {extracted_instructions}" else: - instructions = content + instructions = extracted_instructions else: input_items.append( { diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 00653a5d127..82a23a0f8be 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4762,9 +4762,15 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ litellm_logging_obj=Mock(), ) - assert "instructions" not in first_request - assert "instructions" not in second_request - assert first_request["input"][0] == _system_input_item("You are Claude Code.") + # The leading, list-format top-level system prompt folds into `instructions` (#42171) + # and stays byte-identical across requests, rather than occupying `input[0]`. + assert first_request["instructions"] == "You are Claude Code." + assert second_request["instructions"] == first_request["instructions"] + assert first_request["input"][0] == { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Read inventory.py."}], + } assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"]) assert second_request["input"][len(first_request["input"]) :] == [ {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'}, @@ -4789,6 +4795,98 @@ def test_system_string_after_a_developer_message_stays_in_input_in_client_order( assert input_items[1] == _system_input_item("Be brief.") +def test_leading_system_list_content_folds_into_instructions(): + """A leading system message whose content is a list of text blocks (how a client + attaching cache_control sends it) folds into `instructions` exactly like a plain string + one, instead of surfacing as a `system` input item that some Responses backends reject + outright (#42171).""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}} + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + + assert instructions == "You are a helpful assistant." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + +def test_leading_system_messages_mixed_str_and_list_concatenate_in_order(): + """Several leading system messages, some string and some list content, concatenate in + order the same way an all-string leading run does.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "Be brief."}, + {"role": "system", "content": [{"type": "text", "text": "Answer in French."}]}, + {"role": "system", "content": "Never use emoji."}, + {"role": "user", "content": "Bonjour"}, + ] + ) + + assert instructions == "Be brief. Answer in French. Never use emoji." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Bonjour"}]}, + ] + + +def test_mid_conversation_system_list_content_stays_in_input_after_a_user_turn(): + """Only the leading run folds (#40269): a list-content system message after the first + non-system message stays a positioned input item.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "user", "content": "Read the file."}, + { + "role": "system", + "content": [{"type": "text", "text": "14982391 tokens left"}], + }, + ] + ) + + assert instructions is None + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]}, + _system_input_item("14982391 tokens left"), + ] + + +def test_leading_system_list_with_a_non_text_block_stays_an_input_item(): + """A non-text block (e.g. an image) in a leading system message's list content is never + silently dropped: the whole message stays an input item.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "image_url", "image_url": {"url": "https://example.com/logo.png"}}, + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + + assert instructions is None + assert len(input_items) == 2 + assert input_items[0]["role"] == "system" + assert input_items[0]["content"][0] == {"type": "input_text", "text": "You are a helpful assistant."} + assert input_items[0]["content"][1]["type"] == "input_image" + + def test_map_optional_params_verbosity_merges_into_text(): """Chat verbosity must land on Responses text.verbosity alongside text.format regardless of key order.""" from litellm.completion_extras.litellm_responses_transformation.transformation import ( From cc8e6c7a458cdadf3b305ec7986fb8b0ba3347c9 Mon Sep 17 00:00:00 2001 From: poodle64 Date: Sat, 10 Oct 2026 20:18:20 +1000 Subject: [PATCH 2/4] fix(responses-bridge): keep a system message carrying a prompt cache breakpoint an input item Since this branch was cut, main carries prompt_cache_breakpoint markers through the bridge on input items' content blocks (#42281, #44119, #45500), and its tests expect a leading system message whose block carries one to stay an input item so the marker survives. Folding that message into `instructions` dropped the marker and broke eight of those tests. The fold now skips a leading system message any of whose blocks carries a prompt_cache_breakpoint, alongside the non-text case it already skipped. A block with only cache_control, the shape #42171 reports, still folds. Co-Authored-By: Claude Opus 5.5 --- .../transformation.py | 50 ++++++++++++------- ...responses_transformation_transformation.py | 20 ++++++++ 2 files changed, 52 insertions(+), 18 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c25d7bd9410..d6c73aef03c 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -129,6 +129,35 @@ def _breakpoint_of(part: object) -> object: return content_part.get("prompt_cache_breakpoint") +def _foldable_system_text(block: object) -> str | None: + if isinstance(block, str): + return block + if not isinstance(block, dict): + return None + text_block: Final = cast(dict[str, object], block) # cast-ok: isinstance confirms a content block mapping + text: Final = text_block.get("text") + if text_block.get("type") != "text" or not isinstance(text, str) or _breakpoint_of(text_block) is not None: + return None + return text + + +def _system_text_for_instructions(message: Mapping[str, object]) -> str | None: + """A leading system message's text for ``instructions``, or None to keep it a positioned input item. + + List content folds when every block is plain text, the shape a client attaching cache_control sends. + A non-text block (an image, say) would be lost in a string, and a prompt_cache_breakpoint only rides + on an input item's content block, so either keeps the message an input item. + """ + content: Final = message.get("content", "") + if isinstance(content, str): + return content + if not isinstance(content, list): + return None + blocks: Final = cast(list[object], content) # cast-ok: isinstance confirms a list of content blocks + texts: Final = tuple(text for text in map(_foldable_system_text, blocks) if text is not None) + return " ".join(texts) if len(texts) == len(blocks) else None + + def _pending_audio_breakpoint(pending: object, part: object) -> object: if not _is_audio_input_part(part): return None @@ -536,24 +565,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_call_id = msg.get("tool_call_id") if role == "system": - # Extract system message as instructions, but only within the leading run of - # system messages: a later one stays a positioned input item so its bytes don't - # unsettle the prompt-cache-stable prefix (#40269). - extracted_instructions: str | None = None - if index < leading_system_count: - if isinstance(content, str): - extracted_instructions = content - elif isinstance(content, list) and all( - isinstance(block, str) or (isinstance(block, dict) and block.get("type") == "text") - for block in content - ): - # Every block is plain text (a bare string or a `{"type": "text", ...}` - # dict), the same shape a client attaching cache_control sends; a - # non-text block (an image, say) fails the `all()` above and falls - # through to the input-item branch below instead of losing it silently. - extracted_instructions = " ".join( - block if isinstance(block, str) else block.get("text", "") for block in content - ) + # Only the leading run of system messages folds into instructions: a later one stays + # a positioned input item so its bytes don't unsettle the cached prefix (#40269). + extracted_instructions = _system_text_for_instructions(msg) if index < leading_system_count else None if extracted_instructions is not None: if instructions: # Concatenate multiple system prompts with a space diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 82a23a0f8be..10527607308 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4887,6 +4887,26 @@ def test_leading_system_list_with_a_non_text_block_stays_an_input_item(): assert input_items[0]["content"][1]["type"] == "input_image" +def test_leading_system_list_with_a_prompt_cache_breakpoint_stays_an_input_item(): + """A prompt_cache_breakpoint rides only on an input item's content block, so a leading + system message carrying one stays an input item and keeps its marker rather than folding.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": [_MARKED_SYSTEM_PART]}, + {"role": "user", "content": "hi"}, + ], + keep_prompt_cache_breakpoints=True, + ) + + assert instructions is None + assert input_items == [ + {"type": "message", "role": "system", "content": [{**_MARKED_SYSTEM_PART, "type": "input_text"}]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + def test_map_optional_params_verbosity_merges_into_text(): """Chat verbosity must land on Responses text.verbosity alongside text.format regardless of key order.""" from litellm.completion_extras.litellm_responses_transformation.transformation import ( From b118d8b22ee6e8d6bb44ab6b24291d76d6b2588d Mon Sep 17 00:00:00 2001 From: poodle64 Date: Sat, 10 Oct 2026 20:45:36 +1000 Subject: [PATCH 3/4] fix(responses-bridge): join folded system blocks on newlines; keep a guardrail's rewritten system item in place Three follow-ups from review of the list-content fold: - Folded blocks join on newlines with empty blocks dropped, as the Anthropic-to-Responses adapter joins the same shape, so a multi-block system prompt keeps its section boundaries. - The Responses guardrail handler converts one rewritten chat row through the bridge to patch its input item in place. A lone list-content system row now folds into `instructions` and returns no item, so a guardrail rewriting a system item abandoned the patch and the request lost its system item (an empty `input` for a system-only request). The row now converts as a developer row, whose content maps to the same input parts. - The system-only comment no longer cites non-string content, and the mid-conversation list test is dropped: main already covers that case. Co-Authored-By: Claude Opus 5.5 --- .../transformation.py | 6 +-- .../guardrail_translation/handler.py | 7 +++- ...responses_transformation_transformation.py | 38 +++++++++++++------ ...test_openai_responses_guardrail_handler.py | 23 +++++++++++ 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index d6c73aef03c..3f3cc310858 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -155,7 +155,8 @@ def _system_text_for_instructions(message: Mapping[str, object]) -> str | None: return None blocks: Final = cast(list[object], content) # cast-ok: isinstance confirms a list of content blocks texts: Final = tuple(text for text in map(_foldable_system_text, blocks) if text is not None) - return " ".join(texts) if len(texts) == len(blocks) else None + # Newline-joined, empty blocks dropped, as the Anthropic-to-Responses adapter joins the same shape. + return "\n".join(filter(None, texts)) if len(texts) == len(blocks) else None def _pending_audio_breakpoint(pending: object, part: object) -> object: @@ -791,8 +792,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead - # of instructions, mirroring how non-string system content is already - # handled in convert_chat_completion_messages_to_responses_api. + # of instructions. is_system_only_request: Final = not converted_input_items and converted_instructions is not None target_input_items: Final = ( _without_audio_input_parts(converted_input_items) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index a267b66bb22..da0fc62f962 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -265,7 +265,12 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp rewritten_content: Final = rewritten.get("content") if isinstance(item.get(field), str) and isinstance(rewritten_content, str): return {**item, field: rewritten_content} - rewritten_row: Final = cast("AllMessageValues", rewritten) # cast-ok: guardrails hand back chat-shaped rows + # A lone system row with content would fold into `instructions` and leave no item to patch; as a developer + # row its content converts to the same input parts and stays an item. + row: Final = cast("Mapping[str, object]", rewritten) # cast-ok: isinstance confirms a mapping, keyed by field name + folds: Final = row.get("role") == "system" and row.get("content") is not None + chat_row: Final = {**row, "role": "developer"} if folds else row + rewritten_row: Final = cast("AllMessageValues", chat_row) # cast-ok: guardrails hand back chat-shaped rows converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( [rewritten_row] ) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 10527607308..e186f2588d3 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4840,26 +4840,42 @@ def test_leading_system_messages_mixed_str_and_list_concatenate_in_order(): ] -def test_mid_conversation_system_list_content_stays_in_input_after_a_user_turn(): - """Only the leading run folds (#40269): a list-content system message after the first - non-system message stays a positioned input item.""" +def test_leading_system_list_blocks_join_on_newlines_without_empty_blocks(): + """A multi-block system prompt keeps its section boundaries: blocks join on newlines and + empty blocks drop, as the Anthropic-to-Responses adapter joins the same shape.""" handler: Final = LiteLLMResponsesTransformationHandler() - input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + _, instructions = handler.convert_chat_completion_messages_to_responses_api( [ - {"role": "user", "content": "Read the file."}, { "role": "system", - "content": [{"type": "text", "text": "14982391 tokens left"}], + "content": [ + {"type": "text", "text": "You are Claude Code."}, + {"type": "text", "text": ""}, + {"type": "text", "text": "# Tone\nBe brief.", "cache_control": {"type": "ephemeral"}}, + ], }, + {"role": "user", "content": "hi"}, ] ) - assert instructions is None - assert input_items == [ - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]}, - _system_input_item("14982391 tokens left"), - ] + assert instructions == "You are Claude Code.\n# Tone\nBe brief." + + +def test_system_only_request_with_list_content_stays_a_system_input_item(): + """Responses rejects an empty input, so a request holding only a list-format system + message still goes out as one system input item, as a string one does.""" + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-4o", + messages=[{"role": "system", "content": [{"type": "text", "text": "You are terse."}]}], + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in request + assert request["input"] == [_system_input_item("You are terse.")] def test_leading_system_list_with_a_non_text_block_stays_an_input_item(): diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 8bd242308f7..d3cd2479a46 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3105,6 +3105,29 @@ class TestPatchEdgeBranches: user_items = [item for item in result["input"] if item.get("role") == "user"] assert _texts(user_items[0]) == [COMPRESSED_MARKER] + @pytest.mark.asyncio + @pytest.mark.parametrize("trailing", [[{"role": "user", "content": "What is the codename?"}], []]) + async def test_list_content_system_item_rewrite_patches_in_place(self, trailing): + """A rewritten list-content system item stays an input item, never moved into instructions.""" + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [ + {"role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]}, + *trailing, + ], + } + + result = await handler.process_input_messages( + data, SystemRewriteGuardrail(rewritten_content=[{"type": "text", "text": COMPRESSED_MARKER}]) + ) + + assert "instructions" not in result + assert result["input"] == [ + {"role": "system", "content": [{"type": "input_text", "text": COMPRESSED_MARKER}]}, + *trailing, + ] + def test_item_rewrite_field_ignores_non_string_type(self): from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field From 10d23a60dd3e8867bbc1ea7d0a07694ceb176c7f Mon Sep 17 00:00:00 2001 From: poodle64 Date: Sat, 10 Oct 2026 21:02:17 +1000 Subject: [PATCH 4/4] fix(responses-guardrails): keep a lone system message an input item when the rebuild empties input When a guardrail's rewrite cannot be patched in place, the handler rebuilds the request through the chat-to-Responses converter. A rebuild left holding only the system message folded it into `instructions` over an empty `input`, which the Responses API rejects. That already happened for string content; folding list content made it happen for list content too, where main kept the item. The rebuild now sends that message as a system item, as the chat bridge already does for a system-only request. Co-Authored-By: Claude Opus 5.5 --- .../guardrail_translation/handler.py | 9 ++++++++ ...test_openai_responses_guardrail_handler.py | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index da0fc62f962..23cfceba306 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -505,6 +505,15 @@ def _patch_or_convert_request_fields( input_items, converted_instructions = ( LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages) ) + if not input_items and converted_instructions is not None: + # The Responses API rejects an empty input, so a lone system message stays a system item, as the chat + # bridge sends a system-only request. + system_item: Final = { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": converted_instructions}], + } + return _RequestFields(input=(system_item,), instructions=None) return _RequestFields(input=tuple(input_items), instructions=converted_instructions) diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index d3cd2479a46..3e3080298c9 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3128,6 +3128,27 @@ class TestPatchEdgeBranches: *trailing, ] + @pytest.mark.asyncio + @pytest.mark.parametrize( + "system_content", + ["Answer from the memo only.", [{"type": "input_text", "text": "Answer from the memo only."}]], + ) + async def test_fallback_leaving_only_the_system_message_keeps_it_an_input_item(self, system_content): + """The Responses API rejects an empty input, so a rebuild left holding only the system + message sends it as a system item rather than as instructions over an empty input.""" + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [{"role": "system", "content": system_content}, {"role": "user", "content": "memo " * 400}], + } + + result = await handler.process_input_messages(data, DroppingRewriteGuardrail()) + + assert "instructions" not in result + assert result["input"] == [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]} + ] + def test_item_rewrite_field_ignores_non_string_type(self): from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field