mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-10 08:28:25 +00:00
fix(anthropic): order messages after system and tools in /v1/messages request body
The /v1/messages transform built the upstream body with messages as the first key, so the part that changes every turn led the request and the stable system and tools prefix followed it. On Vertex AI that ordering makes follow-up turns miss the prompt cache much more often. #36079 moves messages last for the chat completions path, and this applies the same ordering to the Anthropic Messages path, which Vertex AI inherits
This commit is contained in:
parent
7ae9a1c350
commit
eb1b595068
3 changed files with 40 additions and 3 deletions
|
|
@ -693,12 +693,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
messages = strip_advisor_blocks_from_messages(messages)
|
||||
|
||||
anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
|
||||
messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
||||
drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
|
||||
),
|
||||
max_tokens=max_tokens,
|
||||
model=model,
|
||||
**anthropic_messages_optional_request_params,
|
||||
messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
|
||||
drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
|
||||
),
|
||||
)
|
||||
return dict(anthropic_messages_request)
|
||||
|
||||
|
|
|
|||
|
|
@ -364,3 +364,21 @@ def test_drop_params_forwards_a_non_object_message_item_as_sent_on_the_native_re
|
|||
|
||||
assert request["messages"][1] == 7
|
||||
assert request["messages"][2]["content"][0]["content"] == [{"type": "text", "text": "sunny"}]
|
||||
|
||||
|
||||
def test_messages_request_body_ends_with_messages_after_system_and_tools() -> None:
|
||||
messages: Final = [{"role": "user", "content": "Hello"}]
|
||||
request: Final = AnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="claude-opus-5-5",
|
||||
messages=messages,
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 256,
|
||||
"system": [{"type": "text", "text": "You are terse."}],
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert list(request)[-1] == "messages", f"key order sent upstream: {list(request)}"
|
||||
assert request["messages"] == messages
|
||||
|
|
|
|||
|
|
@ -784,3 +784,22 @@ def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_f
|
|||
and info.get("supports_mid_conversation_system") is not True
|
||||
]
|
||||
assert missing == []
|
||||
|
||||
|
||||
def test_vertex_messages_request_body_puts_system_and_tools_before_messages() -> None:
|
||||
result: Final = VertexAIPartnerModelsAnthropicMessagesConfig().transform_anthropic_messages_request(
|
||||
model="claude-opus-5-5",
|
||||
messages=[{"role": "user", "content": "Hello"}],
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 256,
|
||||
"system": [{"type": "text", "text": "You are terse."}],
|
||||
"tools": [{"name": "get_weather", "input_schema": {"type": "object", "properties": {}}}],
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
keys: Final = list(result)
|
||||
assert keys.index("system") < keys.index("messages"), f"key order sent to Vertex: {keys}"
|
||||
assert keys.index("tools") < keys.index("messages"), f"key order sent to Vertex: {keys}"
|
||||
assert result["messages"] == [{"role": "user", "content": "Hello"}]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue