Merge 10d23a60dd into b6f91a221b

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -134,6 +134,36 @@ def _breakpoint_of(part: object) -> object:
     return content_part.get("prompt_cache_breakpoint")
 
 
+def _foldable_system_text(block: object) -> str | None:
+    if isinstance(block, str):
+        return block
+    if not isinstance(block, dict):
+        return None
+    text_block: Final = cast(dict[str, object], block)  # cast-ok: isinstance confirms a content block mapping
+    text: Final = text_block.get("text")
+    if text_block.get("type") != "text" or not isinstance(text, str) or _breakpoint_of(text_block) is not None:
+        return None
+    return text
+
+
+def _system_text_for_instructions(message: Mapping[str, object]) -> str | None:
+    """A leading system message's text for ``instructions``, or None to keep it a positioned input item.
+
+    List content folds when every block is plain text, the shape a client attaching cache_control sends.
+    A non-text block (an image, say) would be lost in a string, and a prompt_cache_breakpoint only rides
+    on an input item's content block, so either keeps the message an input item.
+    """
+    content: Final = message.get("content", "")
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return None
+    blocks: Final = cast(list[object], content)  # cast-ok: isinstance confirms a list of content blocks
+    texts: Final = tuple(text for text in map(_foldable_system_text, blocks) if text is not None)
+    # Newline-joined, empty blocks dropped, as the Anthropic-to-Responses adapter joins the same shape.
+    return "\n".join(filter(None, texts)) if len(texts) == len(blocks) else None
+
+
 def _pending_audio_breakpoint(pending: object, part: object) -> object:
     if not _is_audio_input_part(part):
         return None
@@ -850,13 +880,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tool_call_id = msg.get("tool_call_id")
 
             if role == "system":
-                # Extract system message as instructions
-                if isinstance(content, str) and index < leading_system_count:
+                # Only the leading run of system messages folds into instructions: a later one stays
+                # a positioned input item so its bytes don't unsettle the cached prefix (#40269).
+                extracted_instructions = _system_text_for_instructions(msg) if index < leading_system_count else None
+                if extracted_instructions is not None:
                     if instructions:
                         # Concatenate multiple system prompts with a space
-                        instructions = f"{instructions} {content}"
+                        instructions = f"{instructions} {extracted_instructions}"
                     else:
-                        instructions = content
+                        instructions = extracted_instructions
                 else:
                     input_items.append(
                         {
@@ -1084,8 +1116,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
         # OpenAI's Responses API rejects an empty input. For a system-only
         # request, carry the system message as a system-role input item instead
-        # of instructions, mirroring how non-string system content is already
-        # handled in convert_chat_completion_messages_to_responses_api.
+        # of instructions.
         is_system_only_request: Final = not converted_input_items and converted_instructions is not None
         target_input_items: Final = (
             _without_audio_input_parts(converted_input_items)

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -298,7 +298,12 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp
     rewritten_content: Final = rewritten.get("content")
     if isinstance(item.get(field), str) and isinstance(rewritten_content, str):
         return {**item, field: rewritten_content}
-    rewritten_row: Final = cast("AllMessageValues", rewritten)  # cast-ok: guardrails hand back chat-shaped rows
+    # A lone system row with content would fold into `instructions` and leave no item to patch; as a developer
+    # row its content converts to the same input parts and stays an item.
+    row: Final = cast("Mapping[str, object]", rewritten)  # cast-ok: isinstance confirms a mapping, keyed by field name
+    folds: Final = row.get("role") == "system" and row.get("content") is not None
+    chat_row: Final = {**row, "role": "developer"} if folds else row
+    rewritten_row: Final = cast("AllMessageValues", chat_row)  # cast-ok: guardrails hand back chat-shaped rows
     converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
         [rewritten_row]
     )
@@ -579,6 +584,15 @@ def _patch_or_convert_request_fields(
     input_items, converted_instructions = (
         LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
     )
+    if not input_items and converted_instructions is not None:
+        # The Responses API rejects an empty input, so a lone system message stays a system item, as the chat
+        # bridge sends a system-only request.
+        system_item: Final = {
+            "type": "message",
+            "role": "system",
+            "content": [{"type": "input_text", "text": converted_instructions}],
+        }
+        return _RequestFields(input=(system_item,), instructions=None)
     return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
 
 

tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py

@@ -5464,9 +5464,15 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ
         litellm_logging_obj=Mock(),
     )
 
-    assert "instructions" not in first_request
-    assert "instructions" not in second_request
-    assert first_request["input"][0] == _system_input_item("You are Claude Code.")
+    # The leading, list-format top-level system prompt folds into `instructions` (#42171)
+    # and stays byte-identical across requests, rather than occupying `input[0]`.
+    assert first_request["instructions"] == "You are Claude Code."
+    assert second_request["instructions"] == first_request["instructions"]
+    assert first_request["input"][0] == {
+        "type": "message",
+        "role": "user",
+        "content": [{"type": "input_text", "text": "Read inventory.py."}],
+    }
     assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"])
     assert second_request["input"][len(first_request["input"]) :] == [
         {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'},
@@ -5491,6 +5497,134 @@ def test_system_string_after_a_developer_message_stays_in_input_in_client_order(
     assert input_items[1] == _system_input_item("Be brief.")
 
 
+def test_leading_system_list_content_folds_into_instructions():
+    """A leading system message whose content is a list of text blocks (how a client
+    attaching cache_control sends it) folds into `instructions` exactly like a plain string
+    one, instead of surfacing as a `system` input item that some Responses backends reject
+    outright (#42171)."""
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}}
+                ],
+            },
+            {"role": "user", "content": "hi"},
+        ]
+    )
+
+    assert instructions == "You are a helpful assistant."
+    assert input_items == [
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
+    ]
+
+
+def test_leading_system_messages_mixed_str_and_list_concatenate_in_order():
+    """Several leading system messages, some string and some list content, concatenate in
+    order the same way an all-string leading run does."""
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": "Be brief."},
+            {"role": "system", "content": [{"type": "text", "text": "Answer in French."}]},
+            {"role": "system", "content": "Never use emoji."},
+            {"role": "user", "content": "Bonjour"},
+        ]
+    )
+
+    assert instructions == "Be brief. Answer in French. Never use emoji."
+    assert input_items == [
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Bonjour"}]},
+    ]
+
+
+def test_leading_system_list_blocks_join_on_newlines_without_empty_blocks():
+    """A multi-block system prompt keeps its section boundaries: blocks join on newlines and
+    empty blocks drop, as the Anthropic-to-Responses adapter joins the same shape."""
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    _, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": "You are Claude Code."},
+                    {"type": "text", "text": ""},
+                    {"type": "text", "text": "# Tone\nBe brief.", "cache_control": {"type": "ephemeral"}},
+                ],
+            },
+            {"role": "user", "content": "hi"},
+        ]
+    )
+
+    assert instructions == "You are Claude Code.\n# Tone\nBe brief."
+
+
+def test_system_only_request_with_list_content_stays_a_system_input_item():
+    """Responses rejects an empty input, so a request holding only a list-format system
+    message still goes out as one system input item, as a string one does."""
+    request: Final = LiteLLMResponsesTransformationHandler().transform_request(
+        model="gpt-4o",
+        messages=[{"role": "system", "content": [{"type": "text", "text": "You are terse."}]}],
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert "instructions" not in request
+    assert request["input"] == [_system_input_item("You are terse.")]
+
+
+def test_leading_system_list_with_a_non_text_block_stays_an_input_item():
+    """A non-text block (e.g. an image) in a leading system message's list content is never
+    silently dropped: the whole message stays an input item."""
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {
+                "role": "system",
+                "content": [
+                    {"type": "text", "text": "You are a helpful assistant."},
+                    {"type": "image_url", "image_url": {"url": "https://example.com/logo.png"}},
+                ],
+            },
+            {"role": "user", "content": "hi"},
+        ]
+    )
+
+    assert instructions is None
+    assert len(input_items) == 2
+    assert input_items[0]["role"] == "system"
+    assert input_items[0]["content"][0] == {"type": "input_text", "text": "You are a helpful assistant."}
+    assert input_items[0]["content"][1]["type"] == "input_image"
+
+
+def test_leading_system_list_with_a_prompt_cache_breakpoint_stays_an_input_item():
+    """A prompt_cache_breakpoint rides only on an input item's content block, so a leading
+    system message carrying one stays an input item and keeps its marker rather than folding."""
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": [_MARKED_SYSTEM_PART]},
+            {"role": "user", "content": "hi"},
+        ],
+        keep_prompt_cache_breakpoints=True,
+    )
+
+    assert instructions is None
+    assert input_items == [
+        {"type": "message", "role": "system", "content": [{**_MARKED_SYSTEM_PART, "type": "input_text"}]},
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]},
+    ]
+
+
 def test_map_optional_params_verbosity_merges_into_text():
     """Chat verbosity must land on Responses text.verbosity alongside text.format regardless of key order."""
     from litellm.completion_extras.litellm_responses_transformation.transformation import (

tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py

@@ -3517,6 +3517,50 @@ class TestPatchEdgeBranches:
         user_items = [item for item in result["input"] if item.get("role") == "user"]
         assert _texts(user_items[0]) == [COMPRESSED_MARKER]
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("trailing", [[{"role": "user", "content": "What is the codename?"}], []])
+    async def test_list_content_system_item_rewrite_patches_in_place(self, trailing):
+        """A rewritten list-content system item stays an input item, never moved into instructions."""
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                {"role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]},
+                *trailing,
+            ],
+        }
+
+        result = await handler.process_input_messages(
+            data, SystemRewriteGuardrail(rewritten_content=[{"type": "text", "text": COMPRESSED_MARKER}])
+        )
+
+        assert "instructions" not in result
+        assert result["input"] == [
+            {"role": "system", "content": [{"type": "input_text", "text": COMPRESSED_MARKER}]},
+            *trailing,
+        ]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "system_content",
+        ["Answer from the memo only.", [{"type": "input_text", "text": "Answer from the memo only."}]],
+    )
+    async def test_fallback_leaving_only_the_system_message_keeps_it_an_input_item(self, system_content):
+        """The Responses API rejects an empty input, so a rebuild left holding only the system
+        message sends it as a system item rather than as instructions over an empty input."""
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "input": [{"role": "system", "content": system_content}, {"role": "user", "content": "memo " * 400}],
+        }
+
+        result = await handler.process_input_messages(data, DroppingRewriteGuardrail())
+
+        assert "instructions" not in result
+        assert result["input"] == [
+            {"type": "message", "role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]}
+        ]
+
     def test_item_rewrite_field_ignores_non_string_type(self):
         from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field