From 3448175184d8b68dbcf9ec007d62f85651bb0f41 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:44:49 -0700 Subject: [PATCH] fix(responses bridge): keep mid-conversation system messages in input Only the leading run of system messages (before the first non-system message) is joined into the Responses `instructions` field. A system message that arrives after a user, assistant, or tool turn now becomes a system input item at its position, whether its content is a string or a list, so a client that re-sends the same reminder as a string on the next request produces byte-identical input and `instructions` stays stable. Claude Code >= 2.1.237 appends such reminders after every user turn, and folding them into `instructions` made Azure treat every request as a cold prompt (cached_tokens 0 on every request of a session). Fixes #40198 --- .../transformation.py | 9 +- ...responses_transformation_transformation.py | 140 ++++++++++++++++++ 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 87350b5479c..4fe069b0b7d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -370,7 +370,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): and isinstance(tool_call.get("custom"), dict) ) - for msg in messages: + leading_system_count: Final = next( + (index for index, msg in enumerate(messages) if msg.get("role") != "system"), + len(messages), + ) + + for index, msg in enumerate(messages): role = msg.get("role") content = msg.get("content", "") tool_calls = msg.get("tool_calls") @@ -378,7 +383,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if role == "system": # Extract system message as instructions - if isinstance(content, str): + if isinstance(content, str) and index < leading_system_count: if instructions: # Concatenate multiple system prompts with a space instructions = f"{instructions} {content}" diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index d4d47b145d1..e4ada0a9b31 100644 --- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -4147,3 +4147,143 @@ def test_streaming_final_chunk_carries_provider_metadata(): assert chunks[-1]["content_filters"] == content_filters assert "background" not in chunks[-1] assert all("service_tier" not in chunk for chunk in chunks[:-1]) + + +def _system_input_item(text: str) -> dict[str, object]: + return {"type": "message", "role": "system", "content": [{"type": "input_text", "text": text}]} + + +def test_mid_conversation_system_string_stays_in_input_after_a_user_turn(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Read the file."}, + {"role": "system", "content": "14982391 tokens left"}, + {"role": "user", "content": "Now summarize it."}, + ] + ) + + assert instructions == "You are a helpful assistant." + assert input_items == [ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]}, + _system_input_item("14982391 tokens left"), + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Now summarize it."}]}, + ] + + +def test_leading_system_strings_still_join_instructions_without_a_following_turn(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "system", "content": "Be brief."}, + {"role": "system", "content": "Answer in French."}, + ] + ) + + assert instructions == "Be brief. Answer in French." + assert input_items == [] + + +def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_identical_input_items(): + handler: Final = LiteLLMResponsesTransformationHandler() + reminder: Final = "14982391 tokens left" + + as_string, string_instructions = handler.convert_chat_completion_messages_to_responses_api( + [{"role": "user", "content": "Read the file."}, {"role": "system", "content": reminder}] + ) + as_block, block_instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "user", "content": "Read the file."}, + { + "role": "system", + "content": [{"type": "text", "text": reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + ) + + assert string_instructions is None + assert block_instructions is None + assert json.dumps(as_string) == json.dumps(as_block) + assert as_string[1] == _system_input_item(reminder) + + +def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests(): + handler: Final = LiteLLMResponsesTransformationHandler() + top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}] + first_reminder: Final = "27k chars of deferred tools" + second_reminder: Final = "14982391 tokens left" + first_request_messages: Final = [ + {"role": "system", "content": top_level_system}, + {"role": "user", "content": "Read inventory.py."}, + { + "role": "system", + "content": [{"type": "text", "text": first_reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + second_request_messages: Final = [ + {"role": "system", "content": top_level_system}, + {"role": "user", "content": "Read inventory.py."}, + {"role": "system", "content": first_reminder}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "Read", "arguments": '{"file_path": "inventory.py"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "ITEMS = []"}, + { + "role": "system", + "content": [{"type": "text", "text": second_reminder, "cache_control": {"type": "ephemeral"}}], + }, + ] + + first_request: Final = handler.transform_request( + model="gpt-5.6-luna", + messages=first_request_messages, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + second_request: Final = handler.transform_request( + model="gpt-5.6-luna", + messages=second_request_messages, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=Mock(), + ) + + assert "instructions" not in first_request + assert "instructions" not in second_request + assert first_request["input"][0] == _system_input_item("You are Claude Code.") + assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"]) + assert second_request["input"][len(first_request["input"]) :] == [ + {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'}, + {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "ITEMS = []"}]}, + _system_input_item(second_reminder), + ] + + +def test_system_string_after_a_developer_message_stays_in_input_in_client_order(): + handler: Final = LiteLLMResponsesTransformationHandler() + + input_items, instructions = handler.convert_chat_completion_messages_to_responses_api( + [ + {"role": "developer", "content": "Always answer in French."}, + {"role": "system", "content": "Be brief."}, + {"role": "user", "content": "Bonjour"}, + ] + ) + + assert instructions is None + assert [item["role"] for item in input_items] == ["developer", "system", "user"] + assert input_items[1] == _system_input_item("Be brief.")