diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index fad4caa59a0..bd395efbf73 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4661,6 +4661,14 @@ def _sort_bedrock_assistant_content_blocks( return sorted(blocks, key=_sort_key) +def _bedrock_empty_user_turn( + consumed_messages: bool, contents: Sequence[BedrockMessageBlock] +) -> BedrockMessageBlock | None: + if not consumed_messages or (contents and contents[-1]["role"] == "user"): + return None + return BedrockMessageBlock(role="user", content=[BedrockContentBlock(text=_EMPTY_TEXT_PLACEHOLDER)]) + + def _insert_assistant_continue_message( messages: list[BedrockMessageBlock], assistant_continue_message: str | ChatCompletionAssistantMessage | None = None, @@ -5111,6 +5119,8 @@ class BedrockConverseMessagesProcessor: contents[-1]["content"].extend(tool_content) else: contents.append(BedrockMessageBlock(role="user", content=tool_content)) + if (empty_user_turn := _bedrock_empty_user_turn(msg_i > init_msg_i, contents)) is not None: + contents.append(empty_user_turn) assistant_content: list[BedrockContentBlock] = [] ## MERGE CONSECUTIVE ASSISTANT CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": @@ -5524,6 +5534,8 @@ def _bedrock_converse_messages_pt( contents[-1]["content"].extend(tool_content) else: contents.append(BedrockMessageBlock(role="user", content=tool_content)) + if (empty_user_turn := _bedrock_empty_user_turn(msg_i > init_msg_i, contents)) is not None: + contents.append(empty_user_turn) assistant_content: list[BedrockContentBlock] = [] ## MERGE CONSECUTIVE ASSISTANT CONTENT ## while msg_i < len(messages) and messages[msg_i]["role"] == "assistant": diff --git a/tests/integration/providers/test_bedrock_converse_missing_content_wire.py b/tests/integration/providers/test_bedrock_converse_missing_content_wire.py index 3baeebcd74a..48826cb224a 100644 --- a/tests/integration/providers/test_bedrock_converse_missing_content_wire.py +++ b/tests/integration/providers/test_bedrock_converse_missing_content_wire.py @@ -31,6 +31,7 @@ from integration._support.database import read_rows from integration._support.process import owned_proxy_process from integration._support.upstream import _aws_event_frame from integration._support.wire import Reply, Request, Wire, wire_server +from litellm.litellm_core_utils.prompt_templates.factory import _EMPTY_TEXT_PLACEHOLDER from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with from pydantic import JsonValue, TypeAdapter @@ -98,6 +99,7 @@ _TOOLS: Final[tuple[dict[str, JsonValue], ...]] = ( _CONVERSE_QUESTION: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _QUESTION}]} _CONVERSE_ANSWERED: Final[dict[str, JsonValue]] = {"role": "assistant", "content": [{"text": _ANSWERED}]} _CONVERSE_FOLLOW_UP: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _FOLLOW_UP}]} +_CONVERSE_PLACEHOLDER: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _EMPTY_TEXT_PLACEHOLDER}]} _CONVERSE_TOOL_USE: Final[dict[str, JsonValue]] = { "role": "assistant", "content": [{"toolUse": {"toolUseId": "call_1", "name": "get_weather", "input": {"city": "Boston"}}}], @@ -316,7 +318,7 @@ def modify_params_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tu yield owned.gateway, wire -def test_r01_openai_sync_lone_user_without_content_sends_no_converse_block(gateway: Gateway) -> None: +def test_r01_openai_sync_lone_user_without_content_sends_the_placeholder_turn(gateway: Gateway) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire) with openai.OpenAI(base_url=f"{_proxy_url(gateway)}/v1", api_key=gateway.key, max_retries=0) as client: @@ -326,11 +328,11 @@ def test_r01_openai_sync_lone_user_without_content_sends_no_converse_block(gatew assert completion.choices[0].message.content == _ANSWER, completion target, received = _only_received(wire) assert target == _CONVERSE_TARGET, target - assert received["messages"] == [] and "system" not in received, received + assert received["messages"] == [_CONVERSE_PLACEHOLDER] and "system" not in received, received assert _spend_row(completion.id)["status"] == "success" -async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_keeps_the_earlier_turns( +async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_ends_on_the_placeholder_turn( gateway: Gateway, ) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: @@ -347,11 +349,13 @@ async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_k assert completion.choices[0].message.content == _ANSWER, completion target, received = _only_received(wire) assert target == _CONVERSE_TARGET, target - assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED], received + assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED, _CONVERSE_PLACEHOLDER], received assert _spend_row(completion.id)["status"] == "success" -def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_to_done(gateway: Gateway) -> None: +def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_the_placeholder_turn( + gateway: Gateway, +) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire) lines: Final = _stream_lines( @@ -365,7 +369,7 @@ def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_ (identity,) = {chunk["id"] for chunk in chunks} target, received = _only_received(wire) assert target == _STREAM_TARGET, target - assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED], received + assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED, _CONVERSE_PLACEHOLDER], received assert _spend_row(identity)["status"] == "success" @@ -423,11 +427,11 @@ def test_r08_assistant_without_content_between_two_user_turns_merges_them(gatewa ) -def test_r09_lone_user_with_empty_string_content_sends_no_converse_block(gateway: Gateway) -> None: +def test_r09_lone_user_with_empty_string_content_sends_the_placeholder_turn(gateway: Gateway) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire) _, received = _converse_cell(gateway, wire, _chat(model, (_EMPTY_CONTENT_USER,))) - assert received["messages"] == [], received + assert received["messages"] == [_CONVERSE_PLACEHOLDER], received def test_r10_empty_null_and_missing_content_produce_byte_identical_converse_bodies(gateway: Gateway) -> None: @@ -441,7 +445,7 @@ def test_r10_empty_null_and_missing_content_produce_byte_identical_converse_bodi received: Final = wire.drain() assert [unquote(request.target) for request in received] == [_CONVERSE_TARGET] * 3, received assert len({request.body for request in received}) == 1, received - assert json.loads(received[0].body)["messages"] == [], received + assert json.loads(received[0].body)["messages"] == [_CONVERSE_PLACEHOLDER], received for identity in identities: assert _spend_row(identity)["status"] == "success" @@ -655,11 +659,11 @@ def test_s09_unknown_model_with_a_content_less_turn_never_reaches_the_peer(gatew assert wire.drain() == (), response.text -def test_s10_a_deployment_continue_message_without_content_adds_no_converse_block(gateway: Gateway) -> None: +def test_s10_a_deployment_continue_message_without_content_sends_the_placeholder_turn(gateway: Gateway) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire, user_continue_message={"role": "user"}) _, received = _converse_cell(gateway, wire, _chat(model, (_NO_CONTENT_USER,))) - assert received["messages"] == [], received + assert received["messages"] == [_CONVERSE_PLACEHOLDER], received def test_s11_a_deployment_continue_message_given_as_a_string_errors_in_the_body_and_leaves_the_proxy_serving( @@ -724,7 +728,7 @@ def _content_less_bodies(gateway: Gateway, wire: Wire, model: str, count: int) - received: Final = wire.drain() assert len(received) == len(identities), (identities, received) bodies: Final = tuple(_JSON.validate_python(json.loads(request.body))["messages"] for request in received) - assert all(body in ([], [_continue_turn(_DEPLOYMENT_CONTINUE)]) for body in bodies), bodies + assert all(body in ([_CONVERSE_PLACEHOLDER], [_continue_turn(_DEPLOYMENT_CONTINUE)]) for body in bodies), bodies return bodies @@ -734,7 +738,7 @@ def test_e03_updating_the_deployment_continue_message_under_traffic_never_breaks ) -> None: with wire_server(_converse_peer) as wire, gateway.scenario() as scenario: model: Final = _converse_deployment(scenario, wire) - assert _content_less_bodies(gateway, wire, model, 4) == ([],) * 4 + assert _content_less_bodies(gateway, wire, model, 4) == ([_CONVERSE_PLACEHOLDER],) * 4 gateway.post( "/model/update", { @@ -752,7 +756,7 @@ def test_e03_updating_the_deployment_continue_message_under_traffic_never_breaks @pytest.mark.parametrize( ("continue_message", "expected"), - ((None, []), ({}, [_continue_turn(_DEFAULT_CONTINUE)])), + ((None, [_CONVERSE_PLACEHOLDER]), ({}, [_continue_turn(_DEFAULT_CONTINUE)])), ids=("e04", "e05"), ) def test_e04_e05_a_null_continue_message_means_absent_and_an_empty_one_means_the_default( diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 53be3608979..3afaa42f574 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1032,6 +1032,100 @@ async def test_bedrock_converse_keeps_dropping_redacted_thinking_for_non_claude_ assert assistant_content == [{"text": "hello"}] +CONVERSE_EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize( + "empty_user_message", + [ + {"role": "user", "content": []}, + {"role": "user", "content": None}, + {"role": "user"}, + {"role": "user", "content": ""}, + {"role": "user", "content": [{"type": "text", "text": ""}]}, + ], + ids=["empty-list", "null", "content-key-dropped", "empty-string", "empty-text-block"], +) + +_CONVERSE_TOOL_CALL_TURN: Final = { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "tooluse_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}], +} + + +async def _converse(messages: Sequence[Mapping[str, object]], use_async: bool) -> Sequence[Mapping[str, object]]: + model: Final = "us.anthropic.claude-opus-5-5" + if use_async: + return await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async( + messages=list(messages), model=model, llm_provider="bedrock" + ) + return _bedrock_converse_messages_pt(messages=list(messages), model=model, llm_provider="bedrock") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@CONVERSE_EMPTY_USER_MESSAGES +async def test_bedrock_converse_keeps_an_empty_trailing_user_turn( + use_async: bool, empty_user_message: Mapping[str, object] +) -> None: + result: Final = await _converse( + [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, empty_user_message], use_async + ) + + assert [message["role"] for message in result] == ["user", "assistant", "user"] + assert result[-1]["content"] == [{"text": _EMPTY_TEXT_PLACEHOLDER}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("empty_user_first", [True, False], ids=["empty-user-then-tool", "tool-then-empty-user"]) +@CONVERSE_EMPTY_USER_MESSAGES +async def test_bedrock_converse_empty_user_message_beside_a_tool_result_keeps_the_tool_result_only( + use_async: bool, empty_user_first: bool, empty_user_message: Mapping[str, object] +) -> None: + tool_result: Final = {"role": "tool", "tool_call_id": "tooluse_1", "content": "ok"} + result: Final = await _converse( + [ + {"role": "user", "content": "hi"}, + _CONVERSE_TOOL_CALL_TURN, + *((empty_user_message, tool_result) if empty_user_first else (tool_result, empty_user_message)), + ], + use_async, + ) + + assert [message["role"] for message in result] == ["user", "assistant", "user"] + assert [list(block) for block in result[-1]["content"]] == [["toolResult"]] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +@CONVERSE_EMPTY_USER_MESSAGES +async def test_bedrock_converse_keeps_turns_alternating_around_an_empty_user_message( + use_async: bool, empty_user_message: Mapping[str, object] +) -> None: + result: Final = await _converse( + [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + empty_user_message, + {"role": "assistant", "content": "anything else?"}, + {"role": "user", "content": "no"}, + ], + use_async, + ) + + assert [message["role"] for message in result] == ["user", "assistant", "user", "assistant", "user"] + assert result[2]["content"] == [{"text": _EMPTY_TEXT_PLACEHOLDER}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_async", [False, True]) +async def test_bedrock_converse_adds_no_placeholder_before_a_leading_assistant_turn(use_async: bool) -> None: + result: Final = await _converse( + [{"role": "assistant", "content": "hello"}, {"role": "user", "content": "hi"}], use_async + ) + + assert [message["role"] for message in result] == ["assistant", "user"] + + @pytest.mark.parametrize( "thinking_block", [ @@ -5746,19 +5840,16 @@ async def test_bedrock_converse_messages_pt_async_lone_content_less_user_turn_co @pytest.mark.parametrize("content_less_user_message", _CONTENT_LESS_USER_MESSAGES) -def test_bedrock_converse_messages_pt_lone_content_less_user_turn_adds_no_block_without_a_continue_message( +def test_bedrock_converse_messages_pt_lone_content_less_user_turn_sends_the_placeholder_without_a_continue_message( content_less_user_message: dict[str, object], monkeypatch: pytest.MonkeyPatch ): monkeypatch.setattr(litellm, "modify_params", False) - assert ( - _bedrock_converse_messages_pt( - messages=validate_and_fix_openai_messages([content_less_user_message]), - model="anthropic.claude-haiku-4-5", - llm_provider="bedrock", - ) - == [] - ) + assert _bedrock_converse_messages_pt( + messages=validate_and_fix_openai_messages([content_less_user_message]), + model="anthropic.claude-haiku-4-5", + llm_provider="bedrock", + ) == [{"role": "user", "content": [{"text": _EMPTY_TEXT_PLACEHOLDER}]}] @pytest.fixture()