diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 0c18ce38b13..76adadb2e1f 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -2582,6 +2582,48 @@ def _with_native_redacted_data( _AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues] +def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool: + return ( + message_field(message, "role") in ("tool", "function") + and message_field(message, "tool_call_id") not in replaced_server_tool_answers + ) + + +def _tool_result_later_in_turn( + messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str] +) -> tuple[bool, ...]: + from_the_end: Final = itertools.accumulate( + reversed(messages), + lambda later, message: ( + message_field(message, "role") in ("user", "tool", "function") + and (later or _client_tool_result(message, replaced_server_tool_answers)) + ), + initial=False, + ) + return tuple(from_the_end)[::-1][1:] + + +def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam: + placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER) + add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message) + return placeholder + + +def _cache_breakpoint_placeholder( + message: AllMessageValues, tool_result_follows: bool +) -> tuple[AnthropicMessagesTextParam, ...]: + if tool_result_follows or not message_field(message, "cache_control"): + return () + return (_empty_text_placeholder(message),) + + +def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None: + empty_user_message: Final = next( + (message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None + ) + return None if empty_user_message is None else _empty_text_placeholder(empty_user_message) + + def anthropic_messages_pt( messages: list[AllMessageValues], model: str, @@ -2636,6 +2678,7 @@ def anthropic_messages_pt( is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False force_base64 = is_bedrock_invoke or is_vertex_ai + tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers) msg_i = 0 while msg_i < len(messages): user_content: list[AnthropicMessagesUserMessageValues] = [] @@ -2648,8 +2691,12 @@ def anthropic_messages_pt( ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage ) = messages[msg_i] if user_message_types_block["role"] == "user": - if isinstance(user_message_types_block["content"], list): - for m in user_message_types_block["content"]: + if not (user_block_content := user_message_types_block.get("content")): + user_content.extend( + _cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i]) + ) + elif isinstance(user_block_content, list): + for m in user_block_content: if m.get("type", "") == "image_url": m = cast(ChatCompletionImageObject, m) format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None @@ -2720,10 +2767,10 @@ def anthropic_messages_pt( _file_content_element, ) ) - elif isinstance(user_message_types_block["content"], str): + elif isinstance(user_block_content, str): _anthropic_content_text_element: AnthropicMessagesTextParam = { "type": "text", - "text": user_message_types_block["content"], + "text": user_block_content, } _content_element = add_cache_control_to_content( anthropic_content_element=_anthropic_content_text_element, @@ -2747,6 +2794,8 @@ def anthropic_messages_pt( if user_content: new_messages.append({"role": "user", "content": user_content}) + elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None: + new_messages.append({"role": "user", "content": [placeholder]}) # Track unique tool IDs in this merge block to avoid duplication unique_tool_ids: set[str] = set() diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py index 4d1e77126a9..fd0096f9f4d 100644 --- a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py +++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py @@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool: def _renders(message: object, answered: frozenset[str]) -> bool: """Whether ``anthropic_messages_pt`` puts a block on the wire for this message. - A tool message always becomes a ``tool_result`` and a user message with string - content always becomes a text block (empty text gets a placeholder). A user list - renders only through parts of a type the converter emits; ``None``, an empty list, - and a list of other parts vanish. Assistant messages follow ``_assistant_renders``. + A tool message always becomes a ``tool_result``, and a user message with string or + missing content (``None``, an empty list) always puts its turn on the wire, an empty + turn getting a placeholder. A user list renders only through parts of a type the + converter emits, so a list of only other parts vanishes. Assistant messages follow + ``_assistant_renders``. """ role: Final = message_field(message, "role") if role in _TOOL_ROLES: @@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool: if role == "assistant": return _assistant_renders(message, answered) content: Final = message_field(message, "content") - return isinstance(content, str) or any( - message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content) + return ( + isinstance(content, str) + or not content + or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)) ) diff --git a/tests/benchmarks/test_prompt_template_benchmarks.py b/tests/benchmarks/test_prompt_template_benchmarks.py new file mode 100644 index 00000000000..c7858771d35 --- /dev/null +++ b/tests/benchmarks/test_prompt_template_benchmarks.py @@ -0,0 +1,21 @@ +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt + +LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES: Final[tuple[Mapping[str, object], ...]] = ( + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + *({"role": "user", "content": [], "cache_control": {"type": "ephemeral"}} for _ in range(1_000)), +) + + +@pytest.mark.benchmark +def test_anthropic_messages_pt_long_run_of_empty_cached_user_messages() -> None: + result: Final = anthropic_messages_pt( + messages=list(LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES), model="claude-opus-5", llm_provider="vertex_ai" + ) + + assert len(result[-1]["content"]) == 1_000 diff --git a/tests/integration/providers/test_anthropic_empty_user_turn_wire.py b/tests/integration/providers/test_anthropic_empty_user_turn_wire.py new file mode 100644 index 00000000000..654a88c60bb --- /dev/null +++ b/tests/integration/providers/test_anthropic_empty_user_turn_wire.py @@ -0,0 +1,216 @@ +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest +import yaml +from integration._support.client import Gateway, gateway_from_environment, object_value +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import ( + anthropic_deployment, + anthropic_peer, + final_text, + new_marker, +) +from pydantic import JsonValue, TypeAdapter + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_MODEL: Final = "empty-user-turn" +_PLACEHOLDER: Final = "[System: Empty message content sanitised to satisfy protocol]" +_REPLY: Final = "Noted." +_EMPTY_ROWS: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType( + { + "list": MappingProxyType({"role": "user", "content": []}), + "null": MappingProxyType({"role": "user", "content": None}), + "missing": MappingProxyType({"role": "user"}), + "empty-string": MappingProxyType({"role": "user", "content": ""}), + } +) +_CHAT_EMPTY_KINDS: Final = ("list", "null", "missing") + +Turn = tuple[str, tuple[str, ...]] + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + provider: Wire + + +def _empty_row(kind: str) -> dict[str, JsonValue]: + return dict(_EMPTY_ROWS[kind]) + + +def _guardrail_peer(request: Request) -> Reply: + kind: Final = request.target.split("/")[1] + rows: Final = _JSON_OBJECT.validate_json(request.body)["structured_messages"] + assert isinstance(rows, list), rows + verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [*rows, _empty_row(kind)]} + return Reply(body=json.dumps(verdict).encode()) + + +def _guardrail(kind: str, api_base: str) -> dict[str, JsonValue]: + return { + "guardrail_name": f"append-{kind}", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{api_base}/{kind}", + "api_key": "synthetic-guardrail-key", + }, + } + + +def _config(directory: Path, provider_url: str, guardrail_url: str) -> Path: + stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config: Final[dict[str, JsonValue]] = { + **stock, + "model_list": [anthropic_deployment(_MODEL, provider_url)], + "router_settings": {**object_value(stock["router_settings"]), "num_retries": 0}, + "guardrails": [_guardrail(kind, guardrail_url) for kind in _EMPTY_ROWS], + } + path: Final = directory / "empty-user-turn.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("empty-user-turn") + with ( + wire_server(anthropic_peer) as provider, + wire_server(_guardrail_peer) as guardrail, + gateway_from_environment() as gateway, + ): + config: Final = _config(directory, provider.url, guardrail.url) + with owned_proxy(gateway, directory, {}, config=config, workers=2) as owned: + yield _Rig(owned, provider) + + +def _block(block: JsonValue) -> str: + part: Final = object_value(block) + return str(part["text"]) if part.get("type") == "text" else str(part.get("type")) + + +def _turn(message: JsonValue) -> Turn: + row: Final = object_value(message) + content: Final = row["content"] + if isinstance(content, str): + return (str(row["role"]), (content,)) + assert isinstance(content, list), row + return (str(row["role"]), tuple(_block(block) for block in content)) + + +def _send(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> None: + response: Final = rig.gateway.client.post( + path, json=dict(body), headers={"Authorization": f"Bearer {rig.gateway.key}"} + ) + assert response.status_code == 200, response.text + + +def _provider_turns(rig: _Rig, marker: str) -> tuple[Turn, ...]: + needle: Final = marker.encode() + requests: Final = [request for request in rig.provider.drain() if needle in request.body] + assert len(requests) == 1, requests + messages: Final = _JSON_OBJECT.validate_json(requests[0].body)["messages"] + assert isinstance(messages, list), messages + return tuple(_turn(message) for message in messages) + + +def _ask_row(ask: str) -> dict[str, JsonValue]: + return {"role": "user", "content": ask} + + +def _chat(messages: Sequence[JsonValue]) -> dict[str, JsonValue]: + return {"model": _MODEL, "max_tokens": 64, "messages": list(messages)} + + +@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS) +def test_chat_trailing_empty_user_message_keeps_a_user_turn(rig: _Rig, kind: str) -> None: + marker: Final = new_marker() + ask: Final = final_text(marker) + _send( + rig, + "/v1/chat/completions", + _chat([_ask_row(ask), {"role": "assistant", "content": _REPLY}, _empty_row(kind)]), + ) + assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,))) + + +@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS) +def test_chat_empty_user_message_beside_text_sends_only_the_text(rig: _Rig, kind: str) -> None: + marker: Final = new_marker() + ask: Final = final_text(marker) + _send(rig, "/v1/chat/completions", _chat([_ask_row(ask), _empty_row(kind)])) + assert _provider_turns(rig, marker) == (("user", (ask,)),) + + +def _system_then_empty_turns(rig: _Rig, kind: str) -> tuple[Turn, ...]: + marker: Final = new_marker() + ask: Final = final_text(marker) + _send( + rig, + "/v1/chat/completions", + _chat( + [ + _ask_row(ask), + {"role": "assistant", "content": _REPLY}, + {"role": "system", "content": "Answer in French."}, + _empty_row(kind), + ] + ), + ) + turns: Final = _provider_turns(rig, marker) + assert turns[0] == ("user", (ask,)), turns + return turns[1:] + + +@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS) +def test_chat_system_message_before_an_empty_user_message_is_placed_like_an_empty_string(rig: _Rig, kind: str) -> None: + assert _system_then_empty_turns(rig, kind) == _system_then_empty_turns(rig, "empty-string") + + +def test_responses_trailing_empty_input_message_keeps_a_user_turn(rig: _Rig) -> None: + marker: Final = new_marker() + ask: Final = final_text(marker) + body: Final[dict[str, JsonValue]] = { + "model": _MODEL, + "max_output_tokens": 64, + "input": [ + _ask_row(ask), + {"role": "assistant", "content": _REPLY}, + {"role": "user", "content": []}, + ], + } + _send(rig, "/v1/responses", body) + assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,))) + + +@pytest.mark.parametrize("kind", tuple(_EMPTY_ROWS)) +def test_messages_guardrail_empty_user_row_is_sent_like_an_empty_string(rig: _Rig, kind: str) -> None: + marker: Final = new_marker() + ask: Final = final_text(marker) + body: Final[dict[str, JsonValue]] = { + "model": _MODEL, + "max_tokens": 64, + "messages": [_ask_row(ask)], + "guardrails": [f"append-{kind}"], + } + _send(rig, "/v1/messages", body) + assert _provider_turns(rig, marker) == (("user", (ask,)), ("user", (_PLACEHOLDER,))) + + +def test_a_long_run_of_empty_user_messages_is_sent_as_one_text_turn(rig: _Rig) -> None: + marker: Final = new_marker() + ask: Final = final_text(marker) + _send( + rig, "/v1/chat/completions", _chat([*({"role": "user", "content": []} for _ in range(10_000)), _ask_row(ask)]) + ) + assert _provider_turns(rig, marker) == (("user", (ask,)),) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index dba1730208d..ad216548d08 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -4,9 +4,9 @@ import logging import os import re import time -from collections.abc import Callable +from collections.abc import Callable, Mapping, Sequence from copy import deepcopy -from typing import Final, List +from typing import Final, List, TypeAlias from unittest.mock import MagicMock, patch import httpx @@ -16,6 +16,7 @@ import respx import litellm from litellm.litellm_core_utils.prompt_templates.factory import ( BEDROCK_DOCUMENT_PLACEHOLDER_TEXT, + _EMPTY_TEXT_PLACEHOLDER, BedrockConverseMessagesProcessor, BedrockImageProcessor, _bedrock_converse_messages_pt, @@ -795,6 +796,138 @@ def test_anthropic_messages_pt_keeps_signed_thinking_block(): assert thinking_blocks[0]["thinking"] == "genuine anthropic reasoning" +EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize( + "empty_user_message", + [{"role": "user", "content": None}, {"role": "user", "content": []}, {"role": "user"}], + ids=["null", "empty-list", "content-key-dropped"], +) + +Message: TypeAlias = Mapping[str, object] + + +def _convert(messages: Sequence[Message]) -> Sequence[Message]: + return anthropic_messages_pt(messages=list(messages), model="claude-opus-5", llm_provider="vertex_ai") + + +@EMPTY_USER_MESSAGES +def test_anthropic_messages_pt_preserves_trailing_user_message_with_empty_content(empty_user_message: Message) -> None: + result: Final = _convert( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + empty_user_message, + ] + ) + + assert [message["role"] for message in result] == ["user", "assistant", "user"] + assert result[-1]["content"] == [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER}] + + +@EMPTY_USER_MESSAGES +@pytest.mark.parametrize( + "cache_control", [{}, {"cache_control": {"type": "ephemeral"}}], ids=["no-cache-control", "with-cache-control"] +) +def test_anthropic_messages_pt_empty_user_message_beside_tool_result_keeps_tool_result_only( + empty_user_message: Message, cache_control: Message +) -> None: + result: Final = _convert( + [ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}], + }, + {**empty_user_message, **cache_control}, + {"role": "tool", "tool_call_id": "toolu_1", "content": "ok"}, + ] + ) + + assert [message["role"] for message in result] == ["user", "assistant", "user"] + assert [block["type"] for block in result[-1]["content"]] == ["tool_result"] + + +@EMPTY_USER_MESSAGES +def test_anthropic_messages_pt_empty_user_message_keeps_cache_control_like_an_empty_string( + empty_user_message: Message, +) -> None: + def last_turn(last_user_message: Message) -> Message: + return _convert( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {**last_user_message, "cache_control": {"type": "ephemeral"}}, + ] + )[-1] + + assert last_turn(empty_user_message)["content"][0]["cache_control"] == {"type": "ephemeral"} + assert last_turn(empty_user_message) == last_turn({"role": "user", "content": ""}) + + +TOOL_CALL_AND_RESULT: Final[tuple[Message, ...]] = ( + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "toolu_1", "content": "ok"}, +) + + +@EMPTY_USER_MESSAGES +@pytest.mark.parametrize( + "messages_before", + [ + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "q"}, + ], + [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, {"role": "user", "content": []}], + TOOL_CALL_AND_RESULT, + ], + ids=["after-user-text", "after-another-empty-message", "after-tool-result"], +) +def test_anthropic_messages_pt_empty_user_message_beside_other_content_sends_nothing_extra( + empty_user_message: Message, messages_before: Sequence[Message] +) -> None: + assert _convert([*messages_before, empty_user_message]) == _convert(messages_before) + + +@EMPTY_USER_MESSAGES +def test_anthropic_messages_pt_empty_user_message_with_cache_control_after_text_keeps_its_breakpoint( + empty_user_message: Message, +) -> None: + def last_turn(last_user_message: Message) -> Message: + return _convert( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "long document"}, + {**last_user_message, "cache_control": {"type": "ephemeral"}}, + ] + )[-1] + + assert last_turn(empty_user_message)["content"][-1]["cache_control"] == {"type": "ephemeral"} + assert last_turn(empty_user_message) == last_turn({"role": "user", "content": ""}) + + +def test_anthropic_messages_pt_keeps_every_breakpoint_of_a_long_run_of_empty_user_messages() -> None: + result: Final = _convert( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + *[{"role": "user", "content": [], "cache_control": {"type": "ephemeral"}}] * 10_000, + ] + ) + + assert ( + result[-1]["content"] + == [{"type": "text", "text": _EMPTY_TEXT_PLACEHOLDER, "cache_control": {"type": "ephemeral"}}] * 10_000 + ) + + def test_convert_to_azure_openai_messages(): """Test coverting image_url to azure_openai spec""" @@ -4933,7 +5066,9 @@ def test_bedrock_converse_messages_pt_user_message_without_content_adds_no_block assert _bedrock_converse_messages_pt( messages=with_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" - ) == _bedrock_converse_messages_pt(messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock") + ) == _bedrock_converse_messages_pt( + messages=without_message, model="anthropic.claude-haiku-4-5", llm_provider="bedrock" + ) @pytest.mark.asyncio @@ -5063,6 +5198,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -5115,6 +5251,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -5135,6 +5272,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -5157,6 +5295,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_function_call_non_openai_model(): try: @@ -5185,6 +5324,7 @@ def test_function_call_non_openai_model(): print(e) pass + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_empty_content(): """ @@ -5203,6 +5343,7 @@ def test_empty_content(): litellm_call_id=str(uuid.uuid4()), ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_signature_removal_for_non_gemini(): """ @@ -5251,6 +5392,7 @@ def test_thought_signature_removal_for_non_gemini(): assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[1]["tool_calls"][0]["id"] assert THOUGHT_SIGNATURE_SEPARATOR not in processed_messages[2]["tool_call_id"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_signature_preserved_for_gemini(): """ @@ -5297,6 +5439,7 @@ def test_thought_signature_preserved_for_gemini(): assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[1]["tool_calls"][0]["id"] assert THOUGHT_SIGNATURE_SEPARATOR in processed_messages[2]["tool_call_id"] + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_thought_signature_removal_with_multiple_tool_calls(): """ @@ -5351,6 +5494,7 @@ def test_thought_signature_removal_with_multiple_tool_calls(): assert processed_messages[2]["tool_call_id"] == "call_1" assert processed_messages[3]["tool_call_id"] == "call_2" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") def test_messages_without_tool_calls_unchanged(): """ diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py index 6bc1178f64b..91d7acd331e 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py @@ -29,11 +29,8 @@ def _texts(message: dict) -> list[str]: return [block["text"] for block in message["content"]] -SENDS_NOTHING = pytest.mark.parametrize( - "empty_content", - [[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]], - ids=["empty-list", "none", "unsupported-part-only"], -) +EMPTY_USER_CONTENT: Final = pytest.mark.parametrize("empty_content", [[], None], ids=["empty-list", "none"]) +UNSUPPORTED_PART_ONLY: Final = ({"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},) def test_split_leading_system_run_keeps_later_system_messages_in_the_conversation(): @@ -213,9 +210,9 @@ def test_flagged_placement_of_an_earlier_run_does_not_move_when_later_turns_are_ assert _roles(placed_n_plus_one) == ["user", "assistant", "user", "assistant", "user"] -@SENDS_NOTHING -def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothing(empty_content): - placed = place_mid_conversation_system( +@EMPTY_USER_CONTENT +def test_flagged_placement_anchors_a_run_on_a_preceding_empty_user_turn(empty_content: list[object] | None) -> None: + placed: Final = place_mid_conversation_system( [ {"role": "user", "content": empty_content}, {"role": "system", "content": "reminder"}, @@ -225,18 +222,48 @@ def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothin supports_mid_conversation_system=True, ) + assert _roles(placed) == ["user", "system", "assistant", "user"] + assert _texts(placed[1]) == ["reminder"] + + +@EMPTY_USER_CONTENT +def test_flagged_placement_anchors_a_run_on_a_following_empty_user_turn(empty_content: list[object] | None) -> None: + placed: Final = place_mid_conversation_system( + [ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1"}, + {"role": "system", "content": "reminder"}, + {"role": "user", "content": empty_content}, + ], + supports_mid_conversation_system=True, + ) + + assert _roles(placed) == ["user", "assistant", "user", "system"] + assert _texts(placed[3]) == ["reminder"] + + +def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothing() -> None: + placed: Final = place_mid_conversation_system( + [ + {"role": "user", "content": list(UNSUPPORTED_PART_ONLY)}, + {"role": "system", "content": "reminder"}, + {"role": "assistant", "content": "a1"}, + {"role": "user", "content": "q2"}, + ], + supports_mid_conversation_system=True, + ) + assert _roles(placed) == ["user", "user", "assistant", "user"] assert _texts(placed[1]) == [CONVERTED_SYSTEM_NOTE, "reminder"] -@SENDS_NOTHING -def test_flagged_placement_converts_a_run_whose_following_user_turn_sends_nothing(empty_content): - placed = place_mid_conversation_system( +def test_flagged_placement_converts_a_run_whose_following_user_turn_sends_nothing() -> None: + placed: Final = place_mid_conversation_system( [ {"role": "user", "content": "q1"}, {"role": "assistant", "content": "a1"}, {"role": "system", "content": "reminder"}, - {"role": "user", "content": empty_content}, + {"role": "user", "content": list(UNSUPPORTED_PART_ONLY)}, ], supports_mid_conversation_system=True, ) diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 0c093700822..724e137c835 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -6763,15 +6763,11 @@ def test_chat_flagged_model_converts_system_with_no_following_user_turn(local_mo assert _texts(result["messages"][2]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT] -@pytest.mark.parametrize( - "empty_content", - [[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]], - ids=["empty-list", "none", "unsupported-part-only"], -) -def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_nothing( - local_model_cost_map, empty_content -): - messages = [ +@pytest.mark.parametrize("empty_content", [[], None], ids=["empty-list", "none"]) +def test_chat_flagged_model_anchors_a_system_on_an_empty_user_turn( + local_model_cost_map: None, empty_content: list[object] | None +) -> None: + messages: Final = [ {"role": "system", "content": "You are terse."}, {"role": "user", "content": empty_content}, {"role": "system", "content": REMINDER_TEXT}, @@ -6779,7 +6775,22 @@ def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_noth {"role": "user", "content": "Second question"}, ] - result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages) + result: Final = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages) + + assert [m["role"] for m in result["messages"]] == ["user", "system", "assistant", "user"] + assert _texts(result["messages"][1]) == [REMINDER_TEXT] + + +def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_nothing(local_model_cost_map: None) -> None: + messages: Final = [ + {"role": "system", "content": "You are terse."}, + {"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]}, + {"role": "system", "content": REMINDER_TEXT}, + {"role": "assistant", "content": "First answer"}, + {"role": "user", "content": "Second question"}, + ] + + result: Final = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages) assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user"] assert _texts(result["messages"][0]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]