From d510af55fc69f1adbcd143193a1281a2ca7e833a Mon Sep 17 00:00:00 2001 From: Harshit Kulkarni Date: Thu, 5 Nov 2026 16:06:27 +0530 Subject: [PATCH] fix(vertex): order messages after system and tools for Claude prompt caching (#50070) --- .../anthropic/messages/transformation.py | 6 ++++ ...artner_models_anthropic_messages_config.py | 31 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py index 816301c6637..19d230dd701 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/messages/transformation.py @@ -265,6 +265,12 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert sanitize_vertex_anthropic_output_params(anthropic_messages_request, model) + # Vertex AI Claude prompt caching computes prefix cache hits sequentially. + # Place `messages` at the end so static prefixes (system, tools) precede + # dynamic conversation history across tool-calling loops (#50070). + if "messages" in anthropic_messages_request: + anthropic_messages_request["messages"] = anthropic_messages_request.pop("messages") + return anthropic_messages_request def sign_request( diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index fc57dd13302..0570e997748 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -1229,3 +1229,34 @@ def test_beta_detection_preserves_unrecognized_thinking(thinking: object) -> Non assert params["thinking"] == thinking assert ANTHROPIC_THINKING_DISPLAY_UPDATES_BETA_HEADER not in headers.get("anthropic-beta", "").split(",") + + +def test_vertex_anthropic_transform_messages_ordered_last_for_prompt_caching() -> None: + """Ensure messages is placed after static keys (system, tools) in the request body. + Vertex AI Claude prompt caching evaluates prefix cache hits sequentially; + ordering messages last allows static prefixes to hit across tool-loop rounds (#50070).""" + config = VertexAIPartnerModelsAnthropicMessagesConfig() + messages = [{"role": "user", "content": "hello"}] + optional_params = { + "system": [{"type": "text", "text": "You are a helper", "cache_control": {"type": "ephemeral"}}], + "tools": [{"name": "lookup", "description": "test tool"}], + "max_tokens": 1024, + } + litellm_params = GenericLiteLLMParams() + headers = {"anthropic-version": "vertex-2023-10-16"} + + result = config.transform_anthropic_messages_request( + model="vertex_ai/claude-opus-5-5", + messages=messages, + anthropic_messages_optional_request_params=optional_params, + litellm_params=litellm_params, + headers=headers, + ) + + keys = list(result.keys()) + assert "messages" in keys + assert keys[-1] == "messages" + if "system" in keys: + assert keys.index("system") < keys.index("messages") + if "tools" in keys: + assert keys.index("tools") < keys.index("messages")