diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 98322aecf6a..8af22f9474a 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -134,6 +134,36 @@ def _breakpoint_of(part: object) -> object: return content_part.get("prompt_cache_breakpoint") +def _foldable_system_text(block: object) -> str | None: + if isinstance(block, str): + return block + if not isinstance(block, dict): + return None + text_block: Final = cast(dict[str, object], block) # cast-ok: isinstance confirms a content block mapping + text: Final = text_block.get("text") + if text_block.get("type") != "text" or not isinstance(text, str) or _breakpoint_of(text_block) is not None: + return None + return text + + +def _system_text_for_instructions(message: Mapping[str, object]) -> str | None: + """A leading system message's text for ``instructions``, or None to keep it a positioned input item. + + List content folds when every block is plain text, the shape a client attaching cache_control sends. + A non-text block (an image, say) would be lost in a string, and a prompt_cache_breakpoint only rides + on an input item's content block, so either keeps the message an input item. + """ + content: Final = message.get("content", "") + if isinstance(content, str): + return content + if not isinstance(content, list): + return None + blocks: Final = cast(list[object], content) # cast-ok: isinstance confirms a list of content blocks + texts: Final = tuple(text for text in map(_foldable_system_text, blocks) if text is not None) + # Newline-joined, empty blocks dropped, as the Anthropic-to-Responses adapter joins the same shape. + return "\n".join(filter(None, texts)) if len(texts) == len(blocks) else None + + def _pending_audio_breakpoint(pending: object, part: object) -> object: if not _is_audio_input_part(part): return None @@ -850,13 +880,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): tool_call_id = msg.get("tool_call_id") if role == "system": - # Extract system message as instructions - if isinstance(content, str) and index < leading_system_count: + # Only the leading run of system messages folds into instructions: a later one stays + # a positioned input item so its bytes don't unsettle the cached prefix (#40269). + extracted_instructions = _system_text_for_instructions(msg) if index < leading_system_count else None + if extracted_instructions is not None: if instructions: # Concatenate multiple system prompts with a space - instructions = f"{instructions} {content}" + instructions = f"{instructions} {extracted_instructions}" else: - instructions = content + instructions = extracted_instructions else: input_items.append( { @@ -1084,8 +1116,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) # OpenAI's Responses API rejects an empty input. For a system-only # request, carry the system message as a system-role input item instead - # of instructions, mirroring how non-string system content is already - # handled in convert_chat_completion_messages_to_responses_api. + # of instructions. is_system_only_request: Final = not converted_input_items and converted_instructions is not None target_input_items: Final = ( _without_audio_input_parts(converted_input_items) diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 3ea126c5a2a..4b04866a916 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -298,7 +298,12 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp rewritten_content: Final = rewritten.get("content") if isinstance(item.get(field), str) and isinstance(rewritten_content, str): return {**item, field: rewritten_content} - rewritten_row: Final = cast("AllMessageValues", rewritten) # cast-ok: guardrails hand back chat-shaped rows + # A lone system row with content would fold into `instructions` and leave no item to patch; as a developer + # row its content converts to the same input parts and stays an item. + row: Final = cast("Mapping[str, object]", rewritten) # cast-ok: isinstance confirms a mapping, keyed by field name + folds: Final = row.get("role") == "system" and row.get("content") is not None + chat_row: Final = {**row, "role": "developer"} if folds else row + rewritten_row: Final = cast("AllMessageValues", chat_row) # cast-ok: guardrails hand back chat-shaped rows converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api( [rewritten_row] ) @@ -579,6 +584,15 @@ def _patch_or_convert_request_fields( input_items, converted_instructions = ( LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages) ) + if not input_items and converted_instructions is not None: + # The Responses API rejects an empty input, so a lone system message stays a system item, as the chat + # bridge sends a system-only request. + system_item: Final = { + "type": "message", + "role": "system", + "content": [{"type": "input_text", "text": converted_instructions}], + } + return _RequestFields(input=(system_item,), instructions=None) return _RequestFields(input=tuple(input_items), instructions=converted_instructions) diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 9aa494c1aab..1cfb831a128 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -5464,9 +5464,15 @@ def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requ litellm_logging_obj=Mock(), ) - assert "instructions" not in first_request - assert "instructions" not in second_request - assert first_request["input"][0] == _system_input_item("You are Claude Code.") + # The leading, list-format top-level system prompt folds into `instructions` (#42171) + # and stays byte-identical across requests, rather than occupying `input[0]`. + assert first_request["instructions"] == "You are Claude Code." + assert second_request["instructions"] == first_request["instructions"] + assert first_request["input"][0] == { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Read inventory.py."}], + } assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"]) assert second_request["input"][len(first_request["input"]) :] == [ {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'}, @@ -5491,6 +5497,134 @@ def test_system_string_after_a_developer_message_stays_in_input_in_client_order( assert input_items[1] == _system_input_item("Be brief.") +def test_leading_system_list_content_folds_into_instructions(): + """A leading system message whose content is a list of text blocks (how a client + attaching cache_control sends it) folds into `instructions` exactly like a plain string + one, instead of surfacing as a `system` input item that some Responses backends reject + outright (#42171).""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant.", "cache_control": {"type": "ephemeral"}} + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + + assert instructions == "You are a helpful assistant." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + +def test_leading_system_messages_mixed_str_and_list_concatenate_in_order(): + """Several leading system messages, some string and some list content, concatenate in + order the same way an all-string leading run does.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "Be brief."}, + {"role": "system", "content": [{"type": "text", "text": "Answer in French."}]}, + {"role": "system", "content": "Never use emoji."}, + {"role": "user", "content": "Bonjour"}, + ] + ) + + assert instructions == "Be brief. Answer in French. Never use emoji." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Bonjour"}]}, + ] + + +def test_leading_system_list_blocks_join_on_newlines_without_empty_blocks(): + """A multi-block system prompt keeps its section boundaries: blocks join on newlines and + empty blocks drop, as the Anthropic-to-Responses adapter joins the same shape.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + _, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are Claude Code."}, + {"type": "text", "text": ""}, + {"type": "text", "text": "# Tone\nBe brief.", "cache_control": {"type": "ephemeral"}}, + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + + assert instructions == "You are Claude Code.\n# Tone\nBe brief." + + +def test_system_only_request_with_list_content_stays_a_system_input_item(): + """Responses rejects an empty input, so a request holding only a list-format system + message still goes out as one system input item, as a string one does.""" + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model="gpt-4o", + messages=[{"role": "system", "content": [{"type": "text", "text": "You are terse."}]}], + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in request + assert request["input"] == [_system_input_item("You are terse.")] + + +def test_leading_system_list_with_a_non_text_block_stays_an_input_item(): + """A non-text block (e.g. an image) in a leading system message's list content is never + silently dropped: the whole message stays an input item.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + {"type": "image_url", "image_url": {"url": "https://example.com/logo.png"}}, + ], + }, + {"role": "user", "content": "hi"}, + ] + ) + + assert instructions is None + assert len(input_items) == 2 + assert input_items[0]["role"] == "system" + assert input_items[0]["content"][0] == {"type": "input_text", "text": "You are a helpful assistant."} + assert input_items[0]["content"][1]["type"] == "input_image" + + +def test_leading_system_list_with_a_prompt_cache_breakpoint_stays_an_input_item(): + """A prompt_cache_breakpoint rides only on an input item's content block, so a leading + system message carrying one stays an input item and keeps its marker rather than folding.""" + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": [_MARKED_SYSTEM_PART]}, + {"role": "user", "content": "hi"}, + ], + keep_prompt_cache_breakpoints=True, + ) + + assert instructions is None + assert input_items == [ + {"type": "message", "role": "system", "content": [{**_MARKED_SYSTEM_PART, "type": "input_text"}]}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + + def test_map_optional_params_verbosity_merges_into_text(): """Chat verbosity must land on Responses text.verbosity alongside text.format regardless of key order.""" from litellm.completion_extras.litellm_responses_transformation.transformation import ( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index 2f10fd6fd0d..c102915ab3d 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -3517,6 +3517,50 @@ class TestPatchEdgeBranches: user_items = [item for item in result["input"] if item.get("role") == "user"] assert _texts(user_items[0]) == [COMPRESSED_MARKER] + @pytest.mark.asyncio + @pytest.mark.parametrize("trailing", [[{"role": "user", "content": "What is the codename?"}], []]) + async def test_list_content_system_item_rewrite_patches_in_place(self, trailing): + """A rewritten list-content system item stays an input item, never moved into instructions.""" + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [ + {"role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]}, + *trailing, + ], + } + + result = await handler.process_input_messages( + data, SystemRewriteGuardrail(rewritten_content=[{"type": "text", "text": COMPRESSED_MARKER}]) + ) + + assert "instructions" not in result + assert result["input"] == [ + {"role": "system", "content": [{"type": "input_text", "text": COMPRESSED_MARKER}]}, + *trailing, + ] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "system_content", + ["Answer from the memo only.", [{"type": "input_text", "text": "Answer from the memo only."}]], + ) + async def test_fallback_leaving_only_the_system_message_keeps_it_an_input_item(self, system_content): + """The Responses API rejects an empty input, so a rebuild left holding only the system + message sends it as a system item rather than as instructions over an empty input.""" + handler = OpenAIResponsesHandler() + data = { + "model": "gpt-5.6", + "input": [{"role": "system", "content": system_content}, {"role": "user", "content": "memo " * 400}], + } + + result = await handler.process_input_messages(data, DroppingRewriteGuardrail()) + + assert "instructions" not in result + assert result["input"] == [ + {"type": "message", "role": "system", "content": [{"type": "input_text", "text": "Answer from the memo only."}]} + ] + def test_item_rewrite_field_ignores_non_string_type(self): from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field