mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(bedrock): keep an empty user turn on Converse so Claude does not answer a prefill 400
This commit is contained in:
parent
bad98c4b02
commit
560773a64a
3 changed files with 130 additions and 23 deletions
|
|
@ -4661,6 +4661,14 @@ def _sort_bedrock_assistant_content_blocks(
|
|||
return sorted(blocks, key=_sort_key)
|
||||
|
||||
|
||||
def _bedrock_empty_user_turn(
|
||||
consumed_messages: bool, contents: Sequence[BedrockMessageBlock]
|
||||
) -> BedrockMessageBlock | None:
|
||||
if not consumed_messages or (contents and contents[-1]["role"] == "user"):
|
||||
return None
|
||||
return BedrockMessageBlock(role="user", content=[BedrockContentBlock(text=_EMPTY_TEXT_PLACEHOLDER)])
|
||||
|
||||
|
||||
def _insert_assistant_continue_message(
|
||||
messages: list[BedrockMessageBlock],
|
||||
assistant_continue_message: str | ChatCompletionAssistantMessage | None = None,
|
||||
|
|
@ -5111,6 +5119,8 @@ class BedrockConverseMessagesProcessor:
|
|||
contents[-1]["content"].extend(tool_content)
|
||||
else:
|
||||
contents.append(BedrockMessageBlock(role="user", content=tool_content))
|
||||
if (empty_user_turn := _bedrock_empty_user_turn(msg_i > init_msg_i, contents)) is not None:
|
||||
contents.append(empty_user_turn)
|
||||
assistant_content: list[BedrockContentBlock] = []
|
||||
## MERGE CONSECUTIVE ASSISTANT CONTENT ##
|
||||
while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
|
||||
|
|
@ -5524,6 +5534,8 @@ def _bedrock_converse_messages_pt(
|
|||
contents[-1]["content"].extend(tool_content)
|
||||
else:
|
||||
contents.append(BedrockMessageBlock(role="user", content=tool_content))
|
||||
if (empty_user_turn := _bedrock_empty_user_turn(msg_i > init_msg_i, contents)) is not None:
|
||||
contents.append(empty_user_turn)
|
||||
assistant_content: list[BedrockContentBlock] = []
|
||||
## MERGE CONSECUTIVE ASSISTANT CONTENT ##
|
||||
while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from integration._support.database import read_rows
|
|||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.upstream import _aws_event_frame
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import _EMPTY_TEXT_PLACEHOLDER
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
|
|
@ -98,6 +99,7 @@ _TOOLS: Final[tuple[dict[str, JsonValue], ...]] = (
|
|||
_CONVERSE_QUESTION: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _QUESTION}]}
|
||||
_CONVERSE_ANSWERED: Final[dict[str, JsonValue]] = {"role": "assistant", "content": [{"text": _ANSWERED}]}
|
||||
_CONVERSE_FOLLOW_UP: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _FOLLOW_UP}]}
|
||||
_CONVERSE_PLACEHOLDER: Final[dict[str, JsonValue]] = {"role": "user", "content": [{"text": _EMPTY_TEXT_PLACEHOLDER}]}
|
||||
_CONVERSE_TOOL_USE: Final[dict[str, JsonValue]] = {
|
||||
"role": "assistant",
|
||||
"content": [{"toolUse": {"toolUseId": "call_1", "name": "get_weather", "input": {"city": "Boston"}}}],
|
||||
|
|
@ -316,7 +318,7 @@ def modify_params_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tu
|
|||
yield owned.gateway, wire
|
||||
|
||||
|
||||
def test_r01_openai_sync_lone_user_without_content_sends_no_converse_block(gateway: Gateway) -> None:
|
||||
def test_r01_openai_sync_lone_user_without_content_sends_the_placeholder_turn(gateway: Gateway) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
with openai.OpenAI(base_url=f"{_proxy_url(gateway)}/v1", api_key=gateway.key, max_retries=0) as client:
|
||||
|
|
@ -326,11 +328,11 @@ def test_r01_openai_sync_lone_user_without_content_sends_no_converse_block(gatew
|
|||
assert completion.choices[0].message.content == _ANSWER, completion
|
||||
target, received = _only_received(wire)
|
||||
assert target == _CONVERSE_TARGET, target
|
||||
assert received["messages"] == [] and "system" not in received, received
|
||||
assert received["messages"] == [_CONVERSE_PLACEHOLDER] and "system" not in received, received
|
||||
assert _spend_row(completion.id)["status"] == "success"
|
||||
|
||||
|
||||
async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_keeps_the_earlier_turns(
|
||||
async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_ends_on_the_placeholder_turn(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
|
|
@ -347,11 +349,13 @@ async def test_r02_openai_async_user_with_null_content_after_an_assistant_turn_k
|
|||
assert completion.choices[0].message.content == _ANSWER, completion
|
||||
target, received = _only_received(wire)
|
||||
assert target == _CONVERSE_TARGET, target
|
||||
assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED], received
|
||||
assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED, _CONVERSE_PLACEHOLDER], received
|
||||
assert _spend_row(completion.id)["status"] == "success"
|
||||
|
||||
|
||||
def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_to_done(gateway: Gateway) -> None:
|
||||
def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_the_placeholder_turn(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
lines: Final = _stream_lines(
|
||||
|
|
@ -365,7 +369,7 @@ def test_r03_httpx_raw_sse_user_without_content_after_an_assistant_turn_streams_
|
|||
(identity,) = {chunk["id"] for chunk in chunks}
|
||||
target, received = _only_received(wire)
|
||||
assert target == _STREAM_TARGET, target
|
||||
assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED], received
|
||||
assert received["messages"] == [_CONVERSE_QUESTION, _CONVERSE_ANSWERED, _CONVERSE_PLACEHOLDER], received
|
||||
assert _spend_row(identity)["status"] == "success"
|
||||
|
||||
|
||||
|
|
@ -423,11 +427,11 @@ def test_r08_assistant_without_content_between_two_user_turns_merges_them(gatewa
|
|||
)
|
||||
|
||||
|
||||
def test_r09_lone_user_with_empty_string_content_sends_no_converse_block(gateway: Gateway) -> None:
|
||||
def test_r09_lone_user_with_empty_string_content_sends_the_placeholder_turn(gateway: Gateway) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
_, received = _converse_cell(gateway, wire, _chat(model, (_EMPTY_CONTENT_USER,)))
|
||||
assert received["messages"] == [], received
|
||||
assert received["messages"] == [_CONVERSE_PLACEHOLDER], received
|
||||
|
||||
|
||||
def test_r10_empty_null_and_missing_content_produce_byte_identical_converse_bodies(gateway: Gateway) -> None:
|
||||
|
|
@ -441,7 +445,7 @@ def test_r10_empty_null_and_missing_content_produce_byte_identical_converse_bodi
|
|||
received: Final = wire.drain()
|
||||
assert [unquote(request.target) for request in received] == [_CONVERSE_TARGET] * 3, received
|
||||
assert len({request.body for request in received}) == 1, received
|
||||
assert json.loads(received[0].body)["messages"] == [], received
|
||||
assert json.loads(received[0].body)["messages"] == [_CONVERSE_PLACEHOLDER], received
|
||||
for identity in identities:
|
||||
assert _spend_row(identity)["status"] == "success"
|
||||
|
||||
|
|
@ -655,11 +659,11 @@ def test_s09_unknown_model_with_a_content_less_turn_never_reaches_the_peer(gatew
|
|||
assert wire.drain() == (), response.text
|
||||
|
||||
|
||||
def test_s10_a_deployment_continue_message_without_content_adds_no_converse_block(gateway: Gateway) -> None:
|
||||
def test_s10_a_deployment_continue_message_without_content_sends_the_placeholder_turn(gateway: Gateway) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire, user_continue_message={"role": "user"})
|
||||
_, received = _converse_cell(gateway, wire, _chat(model, (_NO_CONTENT_USER,)))
|
||||
assert received["messages"] == [], received
|
||||
assert received["messages"] == [_CONVERSE_PLACEHOLDER], received
|
||||
|
||||
|
||||
def test_s11_a_deployment_continue_message_given_as_a_string_errors_in_the_body_and_leaves_the_proxy_serving(
|
||||
|
|
@ -724,7 +728,7 @@ def _content_less_bodies(gateway: Gateway, wire: Wire, model: str, count: int) -
|
|||
received: Final = wire.drain()
|
||||
assert len(received) == len(identities), (identities, received)
|
||||
bodies: Final = tuple(_JSON.validate_python(json.loads(request.body))["messages"] for request in received)
|
||||
assert all(body in ([], [_continue_turn(_DEPLOYMENT_CONTINUE)]) for body in bodies), bodies
|
||||
assert all(body in ([_CONVERSE_PLACEHOLDER], [_continue_turn(_DEPLOYMENT_CONTINUE)]) for body in bodies), bodies
|
||||
return bodies
|
||||
|
||||
|
||||
|
|
@ -734,7 +738,7 @@ def test_e03_updating_the_deployment_continue_message_under_traffic_never_breaks
|
|||
) -> None:
|
||||
with wire_server(_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _converse_deployment(scenario, wire)
|
||||
assert _content_less_bodies(gateway, wire, model, 4) == ([],) * 4
|
||||
assert _content_less_bodies(gateway, wire, model, 4) == ([_CONVERSE_PLACEHOLDER],) * 4
|
||||
gateway.post(
|
||||
"/model/update",
|
||||
{
|
||||
|
|
@ -752,7 +756,7 @@ def test_e03_updating_the_deployment_continue_message_under_traffic_never_breaks
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
("continue_message", "expected"),
|
||||
((None, []), ({}, [_continue_turn(_DEFAULT_CONTINUE)])),
|
||||
((None, [_CONVERSE_PLACEHOLDER]), ({}, [_continue_turn(_DEFAULT_CONTINUE)])),
|
||||
ids=("e04", "e05"),
|
||||
)
|
||||
def test_e04_e05_a_null_continue_message_means_absent_and_an_empty_one_means_the_default(
|
||||
|
|
|
|||
|
|
@ -1032,6 +1032,100 @@ async def test_bedrock_converse_keeps_dropping_redacted_thinking_for_non_claude_
|
|||
assert assistant_content == [{"text": "hello"}]
|
||||
|
||||
|
||||
CONVERSE_EMPTY_USER_MESSAGES: Final = pytest.mark.parametrize(
|
||||
"empty_user_message",
|
||||
[
|
||||
{"role": "user", "content": []},
|
||||
{"role": "user", "content": None},
|
||||
{"role": "user"},
|
||||
{"role": "user", "content": ""},
|
||||
{"role": "user", "content": [{"type": "text", "text": ""}]},
|
||||
],
|
||||
ids=["empty-list", "null", "content-key-dropped", "empty-string", "empty-text-block"],
|
||||
)
|
||||
|
||||
_CONVERSE_TOOL_CALL_TURN: Final = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "tooluse_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
|
||||
}
|
||||
|
||||
|
||||
async def _converse(messages: Sequence[Mapping[str, object]], use_async: bool) -> Sequence[Mapping[str, object]]:
|
||||
model: Final = "us.anthropic.claude-opus-5-5"
|
||||
if use_async:
|
||||
return await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
|
||||
messages=list(messages), model=model, llm_provider="bedrock"
|
||||
)
|
||||
return _bedrock_converse_messages_pt(messages=list(messages), model=model, llm_provider="bedrock")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@CONVERSE_EMPTY_USER_MESSAGES
|
||||
async def test_bedrock_converse_keeps_an_empty_trailing_user_turn(
|
||||
use_async: bool, empty_user_message: Mapping[str, object]
|
||||
) -> None:
|
||||
result: Final = await _converse(
|
||||
[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}, empty_user_message], use_async
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["user", "assistant", "user"]
|
||||
assert result[-1]["content"] == [{"text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@pytest.mark.parametrize("empty_user_first", [True, False], ids=["empty-user-then-tool", "tool-then-empty-user"])
|
||||
@CONVERSE_EMPTY_USER_MESSAGES
|
||||
async def test_bedrock_converse_empty_user_message_beside_a_tool_result_keeps_the_tool_result_only(
|
||||
use_async: bool, empty_user_first: bool, empty_user_message: Mapping[str, object]
|
||||
) -> None:
|
||||
tool_result: Final = {"role": "tool", "tool_call_id": "tooluse_1", "content": "ok"}
|
||||
result: Final = await _converse(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
_CONVERSE_TOOL_CALL_TURN,
|
||||
*((empty_user_message, tool_result) if empty_user_first else (tool_result, empty_user_message)),
|
||||
],
|
||||
use_async,
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["user", "assistant", "user"]
|
||||
assert [list(block) for block in result[-1]["content"]] == [["toolResult"]]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
@CONVERSE_EMPTY_USER_MESSAGES
|
||||
async def test_bedrock_converse_keeps_turns_alternating_around_an_empty_user_message(
|
||||
use_async: bool, empty_user_message: Mapping[str, object]
|
||||
) -> None:
|
||||
result: Final = await _converse(
|
||||
[
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
empty_user_message,
|
||||
{"role": "assistant", "content": "anything else?"},
|
||||
{"role": "user", "content": "no"},
|
||||
],
|
||||
use_async,
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["user", "assistant", "user", "assistant", "user"]
|
||||
assert result[2]["content"] == [{"text": _EMPTY_TEXT_PLACEHOLDER}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("use_async", [False, True])
|
||||
async def test_bedrock_converse_adds_no_placeholder_before_a_leading_assistant_turn(use_async: bool) -> None:
|
||||
result: Final = await _converse(
|
||||
[{"role": "assistant", "content": "hello"}, {"role": "user", "content": "hi"}], use_async
|
||||
)
|
||||
|
||||
assert [message["role"] for message in result] == ["assistant", "user"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"thinking_block",
|
||||
[
|
||||
|
|
@ -5746,19 +5840,16 @@ async def test_bedrock_converse_messages_pt_async_lone_content_less_user_turn_co
|
|||
|
||||
|
||||
@pytest.mark.parametrize("content_less_user_message", _CONTENT_LESS_USER_MESSAGES)
|
||||
def test_bedrock_converse_messages_pt_lone_content_less_user_turn_adds_no_block_without_a_continue_message(
|
||||
def test_bedrock_converse_messages_pt_lone_content_less_user_turn_sends_the_placeholder_without_a_continue_message(
|
||||
content_less_user_message: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setattr(litellm, "modify_params", False)
|
||||
|
||||
assert (
|
||||
_bedrock_converse_messages_pt(
|
||||
messages=validate_and_fix_openai_messages([content_less_user_message]),
|
||||
model="anthropic.claude-haiku-4-5",
|
||||
llm_provider="bedrock",
|
||||
)
|
||||
== []
|
||||
)
|
||||
assert _bedrock_converse_messages_pt(
|
||||
messages=validate_and_fix_openai_messages([content_less_user_message]),
|
||||
model="anthropic.claude-haiku-4-5",
|
||||
llm_provider="bedrock",
|
||||
) == [{"role": "user", "content": [{"text": _EMPTY_TEXT_PLACEHOLDER}]}]
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue