mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(anthropic): order messages after system/tools in request body
AnthropicConfig.transform_request built the request dict with messages before optional_params, so system and tools (spread in via **optional_params) landed after messages in the serialized JSON body. On the Vertex AI global endpoint, Anthropic's prompt cache keys off the raw request bytes rather than parsed content, so a stable system/tools prefix only hits the cache on repeat turns when messages, the part that changes every turn, comes last. With messages first, every turn after the first missed the cache and re-wrote the full prefix. Fixes #35908
This commit is contained in:
parent
b66d4e6965
commit
1bf3ffe429
2 changed files with 35 additions and 1 deletions
|
|
@ -1935,10 +1935,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
output_key="top_k",
|
||||
)
|
||||
|
||||
# ``messages`` must be the last key in the serialized request body: Anthropic's
|
||||
# prompt cache (observed on the Vertex AI global endpoint) keys off the raw
|
||||
# request bytes, and a stable ``system``/``tools`` prefix only hits the cache
|
||||
# on repeat turns when ``messages`` (the part that changes every turn) is
|
||||
# ordered after them, not before.
|
||||
data: Final = {
|
||||
"model": model,
|
||||
"messages": anthropic_messages,
|
||||
**optional_params,
|
||||
"messages": anthropic_messages,
|
||||
}
|
||||
|
||||
self._apply_output_config(data=data, model=model, optional_params=optional_params)
|
||||
|
|
|
|||
|
|
@ -916,6 +916,35 @@ def test_anthropic_chat_transform_request_includes_context_management():
|
|||
assert result["context_management"] == _sample_context_management_payload()
|
||||
|
||||
|
||||
def test_anthropic_chat_transform_request_orders_messages_last():
|
||||
config = AnthropicConfig()
|
||||
result = config.transform_request(
|
||||
model="claude-sonnet-4-20250514",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "Hello"},
|
||||
],
|
||||
optional_params={
|
||||
"tools": [
|
||||
{
|
||||
"name": "get_weather",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
}
|
||||
],
|
||||
"max_tokens": 256,
|
||||
},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
keys = list(result.keys())
|
||||
assert "system" in keys
|
||||
assert "tools" in keys
|
||||
assert keys.index("messages") > keys.index("system")
|
||||
assert keys.index("messages") > keys.index("tools")
|
||||
assert keys[-1] == "messages"
|
||||
|
||||
|
||||
def test_anthropic_structured_output_beta_header():
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.utils import return_raw_request
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue