diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index 4d1e77126a9..171c5fc5954 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -29,6 +29,7 @@ prompt cache readable across turns. Messages are handled in OpenAI format; the Anthropic wire shape is built later by ``anthropic_messages_pt``. """ +import os from collections.abc import Iterable, Iterator, Mapping, Sequence from itertools import accumulate, chain, dropwhile, groupby, takewhile from operator import itemgetter @@ -55,6 +56,8 @@ CONVERTED_SYSTEM_NOTE: Final = ( "Operator note (not from the user): the following was originally a mid-conversation system-role reminder." ) +DROP_MID_CONVERSATION_SYSTEM_ENV: Final = "LITELLM_DROP_MIDTURN_SYSTEM" + _USER_TYPE_ROLES: Final = frozenset({"user", "tool", "function"}) _TOOL_ROLES: Final = frozenset({"tool", "function"}) _NO_IDS: Final[frozenset[str]] = frozenset() @@ -104,6 +107,22 @@ def _kind(message: object) -> _MessageKind: return "other" +def drops_mid_conversation_system() -> bool: + """``LITELLM_DROP_MIDTURN_SYSTEM=true`` removes every system message after the leading run + before any placement runs, for operators whose backend must never see one (a strict chat + template) and who would rather lose the reminder than deliver it as a user turn.""" + return os.getenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "").strip().lower() == "true" + + +def drop_mid_conversation_system(messages: Sequence[_Message]) -> tuple[_Message, ...]: + """``messages`` without the system messages that follow the leading run.""" + leading_count: Final = next( + (index for index, message in enumerate(messages) if not is_system_message(message)), + len(messages), + ) + return (*messages[:leading_count], *(m for m in messages[leading_count:] if not is_system_message(m))) + + def split_leading_system_run( messages: Sequence[_Message], ) -> tuple[tuple[_Message, ...], tuple[_Message, ...]]: diff --git a/litellm/llms/anthropic/pass_through/adapters/transformation.py b/litellm/llms/anthropic/pass_through/adapters/transformation.py index d1560c5b2ed..acec2806c7c 100644 --- a/litellm/llms/anthropic/pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/pass_through/adapters/transformation.py @@ -127,6 +127,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + drop_mid_conversation_system, + drops_mid_conversation_system, +) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) @@ -467,7 +471,12 @@ class LiteLLMAnthropicMessagesAdapter: preserve_midturn_system: bool = False, ) -> list[AllMessageValues]: new_messages: Final[list[AllMessageValues]] = [] - replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(messages) + kept_messages: Final = ( + messages + if preserve_midturn_system or not drops_mid_conversation_system() + else drop_mid_conversation_system(messages) + ) + replayable_messages: Final = strip_encrypted_reasoning_blocks_from_anthropic_messages(kept_messages) leading_count: Final = next( (i for i, m in enumerate(replayable_messages) if not is_system_role_message(m)), len(replayable_messages), diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index f3f6bc9cbf1..3174d3eb9c4 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -10,6 +10,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, ) from litellm.litellm_core_utils.prompt_templates.factory import parse_mime_type +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( + drop_mid_conversation_system, + drops_mid_conversation_system, +) from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) @@ -170,8 +174,13 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): - video files from file_id or file_data to video_url - thinking_blocks and non-string reasoning_content on assistant messages are removed, and content lists are converted to strings for vLLM compatibility + - system messages after the leading run are removed when + LITELLM_DROP_MIDTURN_SYSTEM=true, for chat templates that reject them """ - for message in messages: + kept_messages: Final = ( + list(drop_mid_conversation_system(messages)) if drops_mid_conversation_system() else messages + ) + for message in kept_messages: if message["role"] == "assistant": message.pop("thinking_blocks", None) if not isinstance(message.get("reasoning_content"), str): @@ -241,6 +250,6 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): message_content[idx] = self._convert_file_to_video_url(content_item) if is_async: - return super()._transform_messages(messages, model, is_async=cast(Literal[True], True)) + return super()._transform_messages(kept_messages, model, is_async=cast(Literal[True], True)) else: - return super()._transform_messages(messages, model, is_async=cast(Literal[False], False)) + return super()._transform_messages(kept_messages, model, is_async=cast(Literal[False], False)) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index 6bc1178f64b..a0506d4287f 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -15,7 +15,10 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import ( CONVERTED_SYSTEM_NOTE, + DROP_MID_CONVERSATION_SYSTEM_ENV, anthropic_system_blocks, + drop_mid_conversation_system, + drops_mid_conversation_system, place_mid_conversation_system, split_leading_system_run, ) @@ -523,3 +526,32 @@ def test_anthropic_system_blocks_keeps_text_parts_with_their_cache_control_and_d {"type": "text", "text": "one", "cache_control": {"type": "ephemeral", "ttl": "1h"}}, {"type": "text", "text": "two", "cache_control": {"type": "ephemeral"}}, ) + + +def test_drop_mid_conversation_system_keeps_the_leading_run_and_removes_every_later_system_message(): + messages = [ + {"role": "system", "content": "one"}, + {"role": "system", "content": "two"}, + {"role": "user", "content": "q1"}, + {"role": "system", "content": "reminder"}, + {"role": "assistant", "content": "a1"}, + {"role": "system", "content": "trailing"}, + ] + + kept = drop_mid_conversation_system(messages) + + assert [m["content"] for m in kept] == ["one", "two", "q1", "a1"] + assert drop_mid_conversation_system(kept) == kept + + +@pytest.mark.parametrize( + ("value", "expected"), + [("true", True), ("TRUE", True), (" true ", True), ("false", False), ("drop", False), ("1", False), (None, False)], +) +def test_drops_mid_conversation_system_only_on_a_literal_true(monkeypatch, value: str | None, expected: bool): + if value is None: + monkeypatch.delenv(DROP_MID_CONVERSATION_SYSTEM_ENV, raising=False) + else: + monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, value) + + assert drops_mid_conversation_system() is expected diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2f0a7ad070b..045a1da4142 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -7,9 +7,6 @@ from typing import Any, Final, cast import pytest import litellm - - - from litellm.litellm_core_utils.prompt_templates.common_utils import ( TOOL_RESULT_IMAGE_PLACEHOLDER, encrypted_reasoning_signature, @@ -18,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( THOUGHT_SIGNATURE_SEPARATOR, _bedrock_converse_messages_pt, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import DROP_MID_CONVERSATION_SYSTEM_ENV from litellm.llms.anthropic.pass_through.adapters.transformation import ( OPENAI_MAX_TOOL_NAME_LENGTH, AnthropicAdapter, @@ -881,6 +879,43 @@ def test_translate_anthropic_to_openai_converts_claude_code_midturn_system_turn( assert converted["content"][1]["text"] == "Keep answers to one sentence." +@pytest.mark.parametrize("custom_llm_provider", [None, "hosted_vllm"]) +def test_translate_anthropic_to_openai_drops_midturn_system_when_flag_set(monkeypatch, custom_llm_provider: str | None): + """ + With LITELLM_DROP_MIDTURN_SYSTEM=true the harness reminder is removed before placement, so the + outbound request carries neither a system row after index 0 nor the operator-note user turn. + """ + monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "true") + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST}, + custom_llm_provider=custom_llm_provider, + ) + + assert openai_request["messages"] == [ + {"role": "system", "content": [{"type": "text", "text": "You are Claude Code."}]}, + {"role": "user", "content": "say hi"}, + {"role": "assistant", "content": "Hi.", "thinking_blocks": None}, + {"role": "user", "content": "say bye"}, + ] + + +def test_translate_anthropic_to_openai_drop_flag_yields_to_preserve_midturn_system(monkeypatch): + """ + Guardrail and compaction callers pass preserve_midturn_system=True to scan the conversation as + sent; the drop flag must not hide the reminder from them. + """ + monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, "true") + + openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai( + anthropic_message_request={"model": "qwen3.8-27B", **_CLAUDE_CODE_MIDTURN_SYSTEM_REQUEST}, + custom_llm_provider="hosted_vllm", + preserve_midturn_system=True, + ) + + assert [m["role"] for m in openai_request["messages"]] == ["system", "user", "system", "assistant", "user"] + + def test_translate_anthropic_to_openai_keeps_midturn_system_when_target_declares_support(monkeypatch): """ A chat-completions target flagged ``supports_mid_conversation_system`` in the cost map accepts diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..539b76670e3 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -8,6 +8,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET, ) +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import DROP_MID_CONVERSATION_SYSTEM_ENV from litellm.llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig @@ -397,3 +398,33 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +@pytest.mark.parametrize( + ("flag", "expected_roles"), + [("true", ["system", "user", "assistant", "user"]), (None, ["system", "user", "system", "assistant", "user"])], + ids=["drop", "default"], +) +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +@pytest.mark.asyncio +async def test_transform_messages_drops_midturn_system_only_when_flag_set(monkeypatch, flag, expected_roles, is_async): + if flag is None: + monkeypatch.delenv(DROP_MID_CONVERSATION_SYSTEM_ENV, raising=False) + else: + monkeypatch.setenv(DROP_MID_CONVERSATION_SYSTEM_ENV, flag) + messages = [ + {"role": "system", "content": "You are Claude Code."}, + {"role": "user", "content": "say hi"}, + {"role": "system", "content": "Keep answers to one sentence."}, + {"role": "assistant", "content": "Hi."}, + {"role": "user", "content": "say bye"}, + ] + config = HostedVLLMChatConfig() + + transformed = ( + await config._transform_messages(messages, model="qwen3.8-27B", is_async=True) + if is_async + else config._transform_messages(messages, model="qwen3.8-27B") + ) + + assert [m["role"] for m in transformed] == expected_roles