fix(anthropic): order messages after system/tools in request body

AnthropicConfig.transform_request built the request dict with messages
before optional_params, so system and tools (spread in via
**optional_params) landed after messages in the serialized JSON body.

On the Vertex AI global endpoint, Anthropic's prompt cache keys off the
raw request bytes rather than parsed content, so a stable system/tools
prefix only hits the cache on repeat turns when messages, the part that
changes every turn, comes last. With messages first, every turn after
the first missed the cache and re-wrote the full prefix.

Fixes #35908
This commit is contained in:
alabenayed 2026-08-06 15:30:01 +03:00
parent b66d4e6965
commit 1bf3ffe429
2 changed files with 35 additions and 1 deletions

View file

@ -1935,10 +1935,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
output_key="top_k",
)
# ``messages`` must be the last key in the serialized request body: Anthropic's
# prompt cache (observed on the Vertex AI global endpoint) keys off the raw
# request bytes, and a stable ``system``/``tools`` prefix only hits the cache
# on repeat turns when ``messages`` (the part that changes every turn) is
# ordered after them, not before.
data: Final = {
"model": model,
"messages": anthropic_messages,
**optional_params,
"messages": anthropic_messages,
}
self._apply_output_config(data=data, model=model, optional_params=optional_params)

View file

@ -916,6 +916,35 @@ def test_anthropic_chat_transform_request_includes_context_management():
assert result["context_management"] == _sample_context_management_payload()
def test_anthropic_chat_transform_request_orders_messages_last():
config = AnthropicConfig()
result = config.transform_request(
model="claude-sonnet-4-20250514",
messages=[
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": "Hello"},
],
optional_params={
"tools": [
{
"name": "get_weather",
"input_schema": {"type": "object", "properties": {}},
}
],
"max_tokens": 256,
},
litellm_params={},
headers={},
)
keys = list(result.keys())
assert "system" in keys
assert "tools" in keys
assert keys.index("messages") > keys.index("system")
assert keys.index("messages") > keys.index("tools")
assert keys[-1] == "messages"
def test_anthropic_structured_output_beta_header():
from litellm.types.utils import CallTypes
from litellm.utils import return_raw_request