diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 05679bf39ab..b2cef62cc50 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -144,6 +144,76 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): else: return system_param + @staticmethod + def _as_system_content_blocks(value: Any) -> list: + if value is None: + return [] + if isinstance(value, list): + return list(value) + if isinstance(value, str): + return [{"type": "text", "text": value}] + return [value] + + @staticmethod + def _is_system_role_message(message: Any) -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None: + """Move ``role: "system"`` entries out of ``messages`` per the Anthropic + ``/v1/messages`` contract, which the first-party API, Bedrock Invoke, + Vertex, and Azure Foundry all enforce identically. + + A *leading* run of system entries is rejected on every model ("messages.0: + use the top-level 'system' parameter for the initial system prompt") and + must be hoisted into the top-level ``system`` field. Models flagged + ``supports_mid_conversation_system`` in the cost map (Claude 4.8+ and the + 5 family) accept a *mid-conversation* entry (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) in place, where it MUST + stay: hoisting one mutates the ``system`` prefix and invalidates the + prompt cache for the whole message history. Older Claude models reject the + role in every position ("role 'system' is not supported on this model"), + so without the flag every system entry is hoisted to keep the request from + 400-ing. Billing-header system blocks are stripped from the top-level + ``system`` field regardless of whether anything was hoisted. + + Subclasses whose upstream rejects the role opt in by calling this from + their ``transform_anthropic_messages_request``; the first-party Anthropic + path forwards ``messages`` untouched and never calls it.""" + from litellm.utils import _supports_factory + + messages = anthropic_messages_request.get("messages") + if not isinstance(messages, list): + return + if _supports_factory( + model=model, + custom_llm_provider=self.custom_llm_provider, + key="supports_mid_conversation_system", + ): + leading_count = next( + (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + len(messages), + ) + hoisted = messages[:leading_count] + remaining = messages[leading_count:] + else: + hoisted = [m for m in messages if self._is_system_role_message(m)] + remaining = [m for m in messages if not self._is_system_role_message(m)] + if hoisted: + anthropic_messages_request["messages"] = remaining + system_content = [ + block + for source in ( + anthropic_messages_request.get("system"), + *(m.get("content") for m in hoisted), + ) + for block in self._as_system_content_blocks(source) + ] + filtered_system = self._filter_billing_headers_from_system(system_content) + if filtered_system: + anthropic_messages_request["system"] = filtered_system + else: + anthropic_messages_request.pop("system", None) + def get_complete_url( self, api_base: Optional[str], diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 8cee35989af..9b05e754b7f 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -166,5 +166,6 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): litellm_params=litellm_params, headers=headers, ) + self._normalize_system_role_messages(anthropic_messages_request, model=model) self._remove_scope_from_cache_control(anthropic_messages_request) return anthropic_messages_request diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index a00d3ba1363..08c13448d8c 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -87,67 +87,6 @@ class AmazonAnthropicClaudeMessagesConfig( BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) - @staticmethod - def _as_system_content_blocks(value: Any) -> list[Any]: - if value is None: - return [] - if isinstance(value, list): - return list(value) - if isinstance(value, str): - return [{"type": "text", "text": value}] - return [value] - - @staticmethod - def _is_system_role_message(message: Any) -> bool: - return isinstance(message, dict) and message.get("role") == "system" - - def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None: - """Bedrock Invoke validates ``role: "system"`` entries inside ``messages`` - per model. Models carrying ``supports_mid_conversation_system`` in the - cost map (the Opus 4.8 family) only reject a leading run ("messages.0: - use the top-level 'system' parameter for the initial system prompt") and - accept mid-conversation entries (e.g. Claude Code's - ``mid-conversation-system-2026-04-07`` reminders) in place, where they - MUST stay: hoisting one mutates the ``system`` prefix and invalidates the - prompt cache for the entire message history. Older Claude models (Opus - 4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position - ("role 'system' is not supported on this model"), so without the flag - every system entry is hoisted into the top-level ``system`` field. - Billing-header system blocks are stripped from the top-level ``system`` - field regardless of whether anything was hoisted.""" - messages = anthropic_messages_request.get("messages") - if not isinstance(messages, list): - return - if _supports_factory( - model=model, - custom_llm_provider="bedrock", - key="supports_mid_conversation_system", - ): - leading_count = next( - (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), - len(messages), - ) - hoisted = messages[:leading_count] - remaining = messages[leading_count:] - else: - hoisted = [m for m in messages if self._is_system_role_message(m)] - remaining = [m for m in messages if not self._is_system_role_message(m)] - if hoisted: - anthropic_messages_request["messages"] = remaining - system_content = [ - block - for source in ( - anthropic_messages_request.get("system"), - *(m.get("content") for m in hoisted), - ) - for block in self._as_system_content_blocks(source) - ] - filtered_system = self._filter_billing_headers_from_system(system_content) - if filtered_system: - anthropic_messages_request["system"] = filtered_system - else: - anthropic_messages_request.pop("system", None) - def validate_anthropic_messages_environment( self, headers: dict, @@ -696,7 +635,7 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model) + self._normalize_system_role_messages(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index de72795cabc..32aaebab768 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -142,6 +142,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert headers=headers, ) + self._normalize_system_role_messages(anthropic_messages_request, model=model) + self._remove_scope_from_cache_control(anthropic_messages_request) anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ee996198b28..3b6c447e741 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2726,6 +2726,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-fable-5": { + "supports_mid_conversation_system": true, "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, "litellm_provider": "azure_ai", @@ -2756,6 +2757,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-8": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2828,6 +2830,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -36554,6 +36557,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -36584,6 +36588,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -36614,6 +36619,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-8": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -36645,6 +36651,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -36704,6 +36711,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -44237,6 +44245,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b1a87c444c8..cb05edada98 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -2726,6 +2726,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-fable-5": { + "supports_mid_conversation_system": true, "input_cost_per_token": 1e-05, "output_cost_per_token": 5e-05, "litellm_provider": "azure_ai", @@ -2756,6 +2757,7 @@ "supports_max_reasoning_effort": true }, "azure_ai/claude-opus-4-8": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "input_cost_per_token": 5e-06, "output_cost_per_token": 2.5e-05, @@ -2828,6 +2830,7 @@ "supports_vision": true }, "azure_ai/claude-sonnet-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -36645,6 +36648,7 @@ "prompt_cache_min_tokens": 2048 }, "vertex_ai/claude-fable-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -36675,6 +36679,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-fable-5@default": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, "cache_read_input_token_cost": 1e-06, @@ -36705,6 +36710,7 @@ "supports_max_reasoning_effort": true }, "vertex_ai/claude-opus-4-8": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -36736,6 +36742,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-opus-4-8@default": { + "supports_mid_conversation_system": true, "supports_adaptive_thinking": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -36795,6 +36802,7 @@ "prompt_cache_min_tokens": 1024 }, "vertex_ai/claude-sonnet-5": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, @@ -44358,6 +44366,7 @@ } }, "vertex_ai/claude-sonnet-5@default": { + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, "cache_read_input_token_cost": 2e-07, diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml index be8a291c6fc..8e83e09a354 100644 --- a/tests/e2e/coverage_registry/llm_conversational.yaml +++ b/tests/e2e/coverage_registry/llm_conversational.yaml @@ -41,6 +41,10 @@ - {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven} - {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven} +- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven} +- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: unproven} +- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: unproven} - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"} - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"} - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"} diff --git a/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py new file mode 100644 index 00000000000..7a04b044634 --- /dev/null +++ b/tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py @@ -0,0 +1,264 @@ +"""Live e2e: model-aware mid-conversation ``role: "system"`` handling on the +Azure AI Foundry and Vertex AI ``/v1/messages`` paths. + +Azure Foundry and Vertex both serve Claude on the first-party Anthropic Messages +contract, verified live: a mid-conversation ``role: "system"`` reminder is +accepted in place on Claude 4.8+/5 (200) but rejected on Claude 4.7 and older +("role 'system' is not supported on this model", 400), and a *leading* system +entry is rejected on every model ("messages.0: use the top-level 'system' +parameter"). This mirrors Bedrock Invoke (PRs #32578/#32831/#32882); the same +model-gated hoist now runs for these two providers (Kraken Tech RCA gap #3). + +Flagged models (``supports_mid_conversation_system`` in the cost map: Claude +4.8+ and the 5 family) must keep the reminder in ``messages`` so the top-level +``system`` prefix stays byte-identical and the prompt cache written on turn one +is read back in full on turn two. Unflagged models (Claude 4.7 and older) must +have the reminder hoisted into the top-level ``system`` field so the call +returns a completion instead of a provider 400. + +The conversation shape mirrors what Claude Code sends mid-session: a cached +system prompt, a user turn carrying its own ``cache_control`` breakpoint, a +``role: "system"`` reminder, an assistant turn, and a fresh user turn. The +message-turn breakpoint is what makes the cache assertion able to fail: a cache +entry whose prefix spans ``system`` plus message turns is invalidated when the +reminder is hoisted (the ``system`` field mutates and a turn disappears from +``messages``), while an entry ending at the system block itself would survive +the hoist and mask the regression. +""" + +from __future__ import annotations + +import time + +import pytest +from pydantic import BaseModel + +from e2e_config import unique_marker +from e2e_http import Result, unwrap +from endpoints_client import ( + CacheControl, + EndpointsClient, + MessagesResult, + RichMessage, + RichMessagesRequest, + TextBlock, +) +from lifecycle import ResourceManager +from models import LiteLLMParamsBody + +pytestmark = pytest.mark.e2e + +CACHE_PRIMING_DEADLINE_SECONDS = 60.0 +CACHE_PRIMING_INTERVAL_SECONDS = 3.0 + + +def _azure_params(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=model, + api_base="os.environ/AZURE_AI_API_BASE", + api_key="os.environ/AZURE_AI_API_KEY", + ) + + +def _vertex_params(model: str) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=model, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="global", + ) + + +def _cacheable_system_block(marker: str) -> TextBlock: + """A system prompt comfortably above the 1024-token minimum cacheable size, + unique per run so no other run's cache entry can satisfy the read.""" + text = " ".join(f"Reference paragraph {index} for run {marker}." for index in range(300)) + return TextBlock(text=text, cache_control=CacheControl()) + + +def _user_turn(text: str, *, cached: bool = False) -> RichMessage: + block = TextBlock(text=text, cache_control=CacheControl() if cached else None) + return RichMessage(role="user", content=[block]) + + +def _system_reminder_turn() -> RichMessage: + return RichMessage( + role="system", + content=[TextBlock(text="Answer with exactly one word.")], + ) + + +def _post_messages(client: EndpointsClient, key: str, body: RichMessagesRequest) -> Result[MessagesResult]: + return client.gateway.transport.post( + "/v1/messages", + headers=client.gateway.transport.bearer(key), + json=body, + response_type=MessagesResult, + ) + + +def _register_deployment( + client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody +) -> str: + model = f"e2e-midsys-{unique_marker()}" + model_id = client.create_model(model, params) + resources.defer(lambda: client.delete_model(model_id)) + return model + + +def _first_turn_user_text(marker: str) -> str: + """A first user turn heavy enough (hundreds of tokens) that losing its cache + entry is unambiguous in the usage numbers, unique per attempt so priming + retries never depend on the proxy's response cache behavior.""" + notes = " ".join(f"Session note {index} for attempt {marker}." for index in range(100)) + return f"Reply with one word.\n{notes}" + + +class PrimedCache(BaseModel): + first_user_text: str + prefix_read_tokens: int + first_turn_creation_tokens: int + + @property + def full_prefix_tokens(self) -> int: + return self.prefix_read_tokens + self.first_turn_creation_tokens + + +def _prime_prompt_cache( + client: EndpointsClient, key: str, model: str, system_block: TextBlock +) -> PrimedCache: + """Send first-turn calls (fresh cache-marked user turn each attempt, + identical system prefix) until one both reads the system prefix back from + cache and writes its own user-turn chunk, proving the cache is live in both + directions. Only the pre-reminder turn is ever retried here, so retries can + never warm a mutated-prefix cache entry and mask the regression the second + turn asserts on.""" + deadline = time.monotonic() + CACHE_PRIMING_DEADLINE_SECONDS + while True: + user_text = _first_turn_user_text(unique_marker()) + body = RichMessagesRequest( + model=model, + system=[system_block], + messages=[_user_turn(user_text, cached=True)], + ) + usage = unwrap(_post_messages(client, key, body)).usage + if usage.cache_read_input_tokens > 0 and usage.cache_creation_input_tokens > 0: + return PrimedCache( + first_user_text=user_text, + prefix_read_tokens=usage.cache_read_input_tokens, + first_turn_creation_tokens=usage.cache_creation_input_tokens, + ) + if time.monotonic() >= deadline: + pytest.fail( + f"{model}: prompt cache never became readable within " + f"{CACHE_PRIMING_DEADLINE_SECONDS}s (last usage: {usage})" + ) + time.sleep(CACHE_PRIMING_INTERVAL_SECONDS) + + +def _assert_flagged_model_keeps_cache( + client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody +) -> None: + model = _register_deployment(client, resources, params) + key = resources.key(models=[model]) + system_block = _cacheable_system_block(unique_marker()) + + primed = _prime_prompt_cache(client, key, model, system_block) + + reminder_turn_body = RichMessagesRequest( + model=model, + system=[system_block], + messages=[ + _user_turn(primed.first_user_text, cached=True), + _system_reminder_turn(), + RichMessage(role="assistant", content=[TextBlock(text="OK.")]), + _user_turn("Reply with one word again.", cached=True), + ], + ) + second = unwrap(_post_messages(client, key, reminder_turn_body)) + + assert second.text.strip(), f"{model}: reminder turn returned no completion text" + assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, ( + f"{model}: turn with a mid-conversation system reminder read " + f"{second.usage.cache_read_input_tokens} cached tokens, expected at " + f"least the {primed.full_prefix_tokens} cached on turn one " + f"({primed.prefix_read_tokens} system prefix + " + f"{primed.first_turn_creation_tokens} first user turn); the reminder " + f"was hoisted into the top-level system field, which mutates the cached " + f"prefix and re-bills the conversation at cache-write pricing" + ) + + +def _assert_unflagged_model_hoists_and_succeeds( + client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody +) -> None: + model = _register_deployment(client, resources, params) + key = resources.key(models=[model]) + + body = RichMessagesRequest( + model=model, + system=[TextBlock(text="You are terse.")], + messages=[ + _user_turn(f"Say hi. Run {unique_marker()}."), + _system_reminder_turn(), + RichMessage(role="assistant", content=[TextBlock(text="Hi.")]), + _user_turn("Say bye."), + ], + ) + completion = unwrap(_post_messages(client, key, body)) + + assert completion.role == "assistant", f"{model}: unexpected role {completion.role!r}" + assert completion.text.strip(), ( + f"{model}: conversation with a mid-conversation system reminder returned " + f"no text; the reminder was forwarded in place to a model that rejects " + f"role 'system' inside messages instead of being hoisted" + ) + + +class TestAzureFoundryMidConversationSystem: + FLAGGED_MODEL = "azure_ai/claude-opus-4-8" + UNFLAGGED_MODEL = "azure_ai/claude-opus-4-7" + + @pytest.mark.covers( + "llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit", + exercised_on=[], + ) + def test_flagged_model_keeps_prompt_cache_across_system_reminder( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + _assert_flagged_model_keeps_cache(endpoints_client, resources, _azure_params(self.FLAGGED_MODEL)) + + @pytest.mark.covers( + "llm.messages.azure_foundry.mid_conversation_system.nonstream.works", + exercised_on=[], + ) + def test_unflagged_model_hoists_system_reminder_and_succeeds( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + _assert_unflagged_model_hoists_and_succeeds( + endpoints_client, resources, _azure_params(self.UNFLAGGED_MODEL) + ) + + +class TestVertexMidConversationSystem: + FLAGGED_MODEL = "vertex_ai/claude-opus-4-8" + UNFLAGGED_MODEL = "vertex_ai/claude-sonnet-4-6" + + @pytest.mark.covers( + "llm.messages.vertex.mid_conversation_system.nonstream.cache_hit", + exercised_on=[], + ) + def test_flagged_model_keeps_prompt_cache_across_system_reminder( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + _assert_flagged_model_keeps_cache(endpoints_client, resources, _vertex_params(self.FLAGGED_MODEL)) + + @pytest.mark.covers( + "llm.messages.vertex.mid_conversation_system.nonstream.works", + exercised_on=[], + ) + def test_unflagged_model_hoists_system_reminder_and_succeeds( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + _assert_unflagged_model_hoists_and_succeeds( + endpoints_client, resources, _vertex_params(self.UNFLAGGED_MODEL) + ) diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 5e9af6bd34d..00d8625896a 100644 --- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -1,3 +1,5 @@ +import copy +import json import os import sys @@ -387,3 +389,106 @@ def test_messages_thinking_shape_follows_exact_azure_entry_flag(local_model_cost assert thinking.get("type") == "enabled" assert isinstance(thinking.get("budget_tokens"), int) assert "output_config" not in flipped + + +def _azure_transform(model, messages, system=None): + config = AzureAnthropicMessagesConfig() + params = {"max_tokens": 256} + if system is not None: + params["system"] = system + return config.transform_anthropic_messages_request( + model=model, + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params=params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +class TestAzureAnthropicMidConversationSystem: + """Azure AI Foundry serves Claude on the first-party Anthropic /v1/messages + contract: a mid-conversation ``role: "system"`` reminder is accepted in place + on Claude 4.8+/5 but 400s ("role 'system' is not supported on this model") on + older Claude, and a *leading* system entry 400s on every model ("messages.0: + use the top-level 'system' parameter"). These tests pin the model-aware hoist + the config applies so Claude Code sessions neither collapse the prompt cache + on 4.8+ nor hard-fail on 4.7 and older (RCA: Kraken Tech high-spend).""" + + def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map): + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + result = _azure_transform("claude-opus-4-8", messages) + assert result["messages"] == messages + + def test_supported_model_hoists_only_leading_system_run(self, local_model_cost_map): + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + result = _azure_transform("claude-opus-4-8", messages) + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, + ] + + def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map): + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + result = _azure_transform( + "claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}] + ) + assert result["messages"] == [ + {"role": "user", "content": "read the file"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "Base."}, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ] + + +def test_azure_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Exact cost-map hits win over the ``claude-mid-conversation-system`` + fallback rule, so an ``azure_ai`` Claude 4.8+/5 entry missing the flag would + be treated as unsupported and hoist every reminder, collapsing the prompt + cache. Every mapped azure_ai entry the rule matches must carry the flag.""" + import re + + import litellm + + cost_map_path = os.path.join( + os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" + ) + with open(cost_map_path) as f: + cost_map = json.load(f) + rules = cost_map["fallback_generalizations"]["rules"] + pattern = re.compile( + next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"), + re.IGNORECASE, + ) + missing = [ + key + for key, info in cost_map.items() + if isinstance(info, dict) + and info.get("litellm_provider") == "azure_ai" + and pattern.search(key) + and info.get("supports_mid_conversation_system") is not True + ] + assert missing == [] diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index ce770221ceb..f20cc6af6b1 100644 --- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -1,3 +1,6 @@ +import copy +import json +import os from unittest.mock import MagicMock, patch import pytest @@ -565,3 +568,107 @@ def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cos assert thinking.get("type") == "enabled" assert isinstance(thinking.get("budget_tokens"), int) assert "output_config" not in flipped + + +def _vertex_transform(model, messages, system=None): + config = VertexAIPartnerModelsAnthropicMessagesConfig() + params = {"max_tokens": 256} + if system is not None: + params["system"] = system + return config.transform_anthropic_messages_request( + model=model, + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params=params, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + +class TestVertexAnthropicMidConversationSystem: + """Vertex serves Claude on the first-party Anthropic /v1/messages contract: a + mid-conversation ``role: "system"`` reminder is accepted in place on Claude + 4.8+/5 but 400s ("role 'system' is not supported on this model") on older + Claude, and a *leading* system entry 400s on every model ("messages.0: use + the top-level 'system' parameter"). These tests pin the model-aware hoist so + Claude Code sessions neither collapse the prompt cache on 4.8+ nor hard-fail + on 4.7 and older (RCA: Kraken Tech high-spend).""" + + def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map): + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + result = _vertex_transform("claude-opus-4-8", messages) + assert result["messages"] == messages + + def test_supported_model_hoists_only_leading_system_run(self, local_model_cost_map): + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + result = _vertex_transform("claude-opus-4-8", messages) + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, + ] + + def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map): + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + result = _vertex_transform( + "claude-sonnet-4-6", messages, system=[{"type": "text", "text": "Base."}] + ) + assert result["messages"] == [ + {"role": "user", "content": "read the file"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "Base."}, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ] + + +def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag(): + """Exact cost-map hits win over the ``claude-mid-conversation-system`` + fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would + be treated as unsupported and hoist every reminder, collapsing the prompt + cache. Every mapped vertex_ai entry the rule matches must carry the flag.""" + import re + + import litellm + + cost_map_path = os.path.join( + os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json" + ) + with open(cost_map_path) as f: + cost_map = json.load(f) + rules = cost_map["fallback_generalizations"]["rules"] + pattern = re.compile( + next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"), + re.IGNORECASE, + ) + missing = [ + key + for key, info in cost_map.items() + if isinstance(info, dict) + and str(info.get("litellm_provider", "")).startswith("vertex_ai") + and "claude" in key + and pattern.search(key) + and info.get("supports_mid_conversation_system") is not True + ] + assert missing == []