Merge 7a23771a24 into 2590f641f5

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -29,6 +29,7 @@ prompt cache readable across turns. Messages are handled in OpenAI format; the
 Anthropic wire shape is built later by ``anthropic_messages_pt``.
 """
 
+import os
 from collections.abc import Iterable, Iterator, Mapping, Sequence
 from itertools import accumulate, chain, dropwhile, groupby, takewhile
 from operator import itemgetter
@@ -55,6 +56,8 @@ CONVERTED_SYSTEM_NOTE: Final = (
     "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
 )
 
+DROP_MID_CONVERSATION_SYSTEM_ENV: Final = "LITELLM_DROP_MIDTURN_SYSTEM"
+
 _USER_TYPE_ROLES: Final = frozenset({"user", "tool", "function"})
 _TOOL_ROLES: Final = frozenset({"tool", "function"})
 _NO_IDS: Final[frozenset[str]] = frozenset()
@@ -104,6 +107,22 @@ def _kind(message: object) -> _MessageKind:
     return "other"
 
 
+def drops_mid_conversation_system() -> bool:
+    """``LITELLM_DROP_MIDTURN_SYSTEM=true`` removes every system message after the leading run
+    before any placement runs, for operators whose backend must never see one (a strict chat
+    template) and who would rather lose the reminder than deliver it as a user turn."""
+    return os.getenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "").strip().lower() == "true"
+
+
+def drop_mid_conversation_system(messages: Sequence[_Message]) -> tuple[_Message, ...]:
+    """``messages`` without the system messages that follow the leading run."""
+    leading_count: Final = next(
+        (index for index, message in enumerate(messages) if not is_system_message(message)),
+        len(messages),
+    )
+    return (*messages[:leading_count], *(m for m in messages[leading_count:] if not is_system_message(m)))
+
+
 def split_leading_system_run(
     messages: Sequence[_Message],
 ) -> tuple[tuple[_Message, ...], tuple[_Message, ...]]:

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -130,6 +130,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.prompt_templates.factory import (
     THOUGHT_SIGNATURE_SEPARATOR,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+    drop_mid_conversation_system,
+    drops_mid_conversation_system,
+)
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking,
 )
@@ -474,7 +478,12 @@ class LiteLLMAnthropicMessagesAdapter:
         preserve_midturn_system: bool = False,
     ) -> list[AllMessageValues]:
         new_messages: Final[list[AllMessageValues]] = []
-        replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages)
+        kept_messages: Final = (
+            messages
+            if preserve_midturn_system or not drops_mid_conversation_system()
+            else drop_mid_conversation_system(messages)
+        )
+        replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(kept_messages)
         leading_count: Final = next(
             (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)),
             len(replayable_messages),

litellm/llms/hosted_vllm/chat/transformation.py

@@ -10,6 +10,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_image_mime_type_from_url,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+    drop_mid_conversation_system,
+    drops_mid_conversation_system,
+)
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking,
 )
@@ -171,8 +175,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
         - video files from file_id or file_data to video_url
         - thinking_blocks and non-string reasoning_content on assistant messages
           are removed, and content lists are converted to strings for vLLM compatibility
+        - system messages after the leading run are removed when
+          LITELLM_DROP_MIDTURN_SYSTEM=true, for chat templates that reject them
         """
-        for message in messages:
+        kept_messages: Final = (
+            list(drop_mid_conversation_system(messages)) if drops_mid_conversation_system() else messages
+        )
+        for message in kept_messages:
             if message["role"] == "assistant":
                 message.pop("thinking_blocks", None)
                 if not isinstance(message.get("reasoning_content"), str):
@@ -242,6 +251,6 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
                         message_content[idx] = self._convert_file_to_video_url(content_item)
 
         if is_async:
-            return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
+            return super()._transform_messages(kept_messages, model, is_async=cast(Literal[True], True))
         else:
-            return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
+            return super()._transform_messages(kept_messages, model, is_async=cast(Literal[False], False))

tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py

@@ -15,7 +15,10 @@ import litellm
 from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
     CONVERTED_SYSTEM_NOTE,
+    DROP_MID_CONVERSATION_SYSTEM_ENV,
     anthropic_system_blocks,
+    drop_mid_conversation_system,
+    drops_mid_conversation_system,
     place_mid_conversation_system,
     split_leading_system_run,
 )
@@ -554,3 +557,32 @@ def test_anthropic_system_blocks_keeps_text_parts_with_their_cache_control_and_d
         {"type": "text", "text": "one", "cache_control": {"type": "ephemeral", "ttl": "1h"}},
         {"type": "text", "text": "two", "cache_control": {"type": "ephemeral"}},
     )
+
+
+def test_drop_mid_conversation_system_keeps_the_leading_run_and_removes_every_later_system_message():
+    messages = [
+        {"role": "system", "content": "one"},
+        {"role": "system", "content": "two"},
+        {"role": "user", "content": "q1"},
+        {"role": "system", "content": "reminder"},
+        {"role": "assistant", "content": "a1"},
+        {"role": "system", "content": "trailing"},
+    ]
+
+    kept = drop_mid_conversation_system(messages)
+
+    assert [m["content"] for m in kept] == ["one", "two", "q1", "a1"]
+    assert drop_mid_conversation_system(kept) == kept
+
+
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [("true", True), ("TRUE", True), (" true ", True), ("false", False), ("drop", False), ("1", False), (None, False)],
+)
+def test_drops_mid_conversation_system_only_on_a_literal_true(monkeypatch, value: str | None, expected: bool):
+    if value is None:
+        monkeypatch.delenv(DROP_MID_CONVERSATION_SYSTEM_ENV, raising=False)
+    else:
+        monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, value)
+
+    assert drops_mid_conversation_system() is expected

tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py

@@ -10,9 +10,6 @@ import pytest
 import respx
 
 import litellm
-
-
-
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     TOOL_RESULT_IMAGE_PLACEHOLDER,
     encrypted_reasoning_signature,
@@ -21,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     THOUGHT_SIGNATURE_SEPARATOR,
     _bedrock_converse_messages_pt,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import DROP_MID_CONVERSATION_SYSTEM_ENV
 from litellm.llms.anthropic.pass_through.adapters.transformation import (
     OPENAI_MAX_TOOL_NAME_LENGTH,
     AnthropicAdapter,
@@ -884,6 +882,43 @@ def test_translate_anthropic_to_openai_converts_claude_code_midturn_system_turn(
     assert converted["content"][1]["text"] == "<system-reminder>Keep answers to one sentence.</system-reminder>"
 
 
+@pytest.mark.parametrize("custom_llm_provider", [None, "hosted_vllm"])
+def test_translate_anthropic_to_openai_drops_midturn_system_when_flag_set(monkeypatch, custom_llm_provider: str | None):
+    """
+    With LITELLM_DROP_MIDTURN_SYSTEM=true the harness reminder is removed before placement, so the
+    outbound request carries neither a system row after index 0 nor the operator-note user turn.
+    """
+    monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "true")
+
+    openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+        anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
+        custom_llm_provider=custom_llm_provider,
+    )
+
+    assert openai_request["messages"] == [
+        {"role": "system", "content": [{"type": "text", "text": "You are Claude Code."}]},
+        {"role": "user", "content": "say hi"},
+        {"role": "assistant", "content": "Hi.", "thinking_blocks": None},
+        {"role": "user", "content": "say bye"},
+    ]
+
+
+def test_translate_anthropic_to_openai_drop_flag_yields_to_preserve_midturn_system(monkeypatch):
+    """
+    Guardrail and compaction callers pass preserve_midturn_system=True to scan the conversation as
+    sent; the drop flag must not hide the reminder from them.
+    """
+    monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "true")
+
+    openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+        anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST},
+        custom_llm_provider="hosted_vllm",
+        preserve_midturn_system=True,
+    )
+
+    assert [m["role"] for m in openai_request["messages"]] == ["system", "user", "system", "assistant", "user"]
+
+
 def test_translate_anthropic_to_openai_keeps_midturn_system_when_target_declares_support(monkeypatch):
     """
     A chat-completions target flagged ``supports_mid_conversation_system`` in the cost map accepts

tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py

@@ -8,6 +8,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import DROP_MID_CONVERSATION_SYSTEM_ENV
 from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig
 
 
@@ -397,3 +398,33 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
     assert tools[0]["function"]["name"] == "search"
     assert tools[0]["function"]["description"] == "Search docs"
     assert tools[0]["function"]["parameters"] == input_schema
+
+
+@pytest.mark.parametrize(
+    ("flag", "expected_roles"),
+    [("true", ["system", "user", "assistant", "user"]), (None, ["system", "user", "system", "assistant", "user"])],
+    ids=["drop", "default"],
+)
+@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
+@pytest.mark.asyncio
+async def test_transform_messages_drops_midturn_system_only_when_flag_set(monkeypatch, flag, expected_roles, is_async):
+    if flag is None:
+        monkeypatch.delenv(DROP_MID_CONVERSATION_SYSTEM_ENV, raising=False)
+    else:
+        monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, flag)
+    messages = [
+        {"role": "system", "content": "You are Claude Code."},
+        {"role": "user", "content": "say hi"},
+        {"role": "system", "content": "<system-reminder>Keep answers to one sentence.</system-reminder>"},
+        {"role": "assistant", "content": "Hi."},
+        {"role": "user", "content": "say bye"},
+    ]
+    config = HostedVLLMChatConfig()
+
+    transformed = (
+        await config._transform_messages(messages, model="qwen3.8-27B", is_async=True)
+        if is_async
+        else config._transform_messages(messages, model="qwen3.8-27B")
+    )
+
+    assert [m["role"] for m in transformed] == expected_roles