scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

fix(vertex,azure): model-aware mid-conversation system for Claude /v1/messages

Browse source 
Azure AI Foundry and Vertex AI serve Claude on the first-party Anthropic
Messages contract, which was verified live to be byte-identical to
api.anthropic.com: a leading role:"system" entry in messages is rejected on
every model ("messages.0: use the top-level 'system' parameter"), and a
mid-conversation role:"system" reminder is accepted in place on Claude 4.8+/5
but 400s on Claude 4.7 and older ("role 'system' is not supported on this
model"). This is the same contract Bedrock Invoke already handles model-aware
(PRs #32578/#32831/#32882); Vertex and Azure did no hoisting at all, so a Claude
Code session on an older Vertex/Azure Claude model hard-400s on its reminder
turns, and the only thing sparing 4.8+/5 was that nothing was hoisted

Extract Bedrock's model-gated normalization into the shared
AnthropicMessagesConfig base as _normalize_system_role_messages and call it from
the Vertex and Azure messages configs. Flagged models (4.8+/5) hoist only the
leading run of system entries and keep mid-conversation reminders in place so
the top-level system prefix stays byte-identical and the prompt cache is
preserved; unflagged models hoist every system entry so the request returns a
completion instead of a 400

Add supports_mid_conversation_system to the azure_ai and vertex_ai Claude 4.8+/5
cost-map entries. Exact cost-map hits win over the claude-mid-conversation-system
fallback rule, so without the explicit flag those models would be treated as
unsupported and hoist every reminder, collapsing the prompt cache (the exact
customer regression). A per-provider test guards this so future 4.8+/5 entries
cannot silently miss the flag

Closes the Vertex/Azure gap from the customer RCA
This commit is contained in:
mateo-berri 2026-07-17 23:11:14 -04:00
parent 40e914cfa7
commit 23b5b7d199
10 changed files with 572 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
litellm/llms/anthropic/experimental_pass_through/messages/transformation.py
View file

@ -144,6 +144,76 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
else:
return system_param
@staticmethod
def _as_system_content_blocks(value: Any) -> list:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
@staticmethod
def _is_system_role_message(message: Any) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def _normalize_system_role_messages(self, anthropic_messages_request: dict, model: str) -> None:
"""Move ``role: "system"`` entries out of ``messages`` per the Anthropic
``/v1/messages`` contract, which the first-party API, Bedrock Invoke,
Vertex, and Azure Foundry all enforce identically.
A *leading* run of system entries is rejected on every model ("messages.0:
use the top-level 'system' parameter for the initial system prompt") and
must be hoisted into the top-level ``system`` field. Models flagged
``supports_mid_conversation_system`` in the cost map (Claude 4.8+ and the
5 family) accept a *mid-conversation* entry (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) in place, where it MUST
stay: hoisting one mutates the ``system`` prefix and invalidates the
prompt cache for the whole message history. Older Claude models reject the
role in every position ("role 'system' is not supported on this model"),
so without the flag every system entry is hoisted to keep the request from
400-ing. Billing-header system blocks are stripped from the top-level
``system`` field regardless of whether anything was hoisted.
Subclasses whose upstream rejects the role opt in by calling this from
their ``transform_anthropic_messages_request``; the first-party Anthropic
path forwards ``messages`` untouched and never calls it."""
from litellm.utils import _supports_factory
messages = anthropic_messages_request.get("messages")
if not isinstance(messages, list):
return
if _supports_factory(
model=model,
custom_llm_provider=self.custom_llm_provider,
key="supports_mid_conversation_system",
):
leading_count = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
len(messages),
)
hoisted = messages[:leading_count]
remaining = messages[leading_count:]
else:
hoisted = [m for m in messages if self._is_system_role_message(m)]
remaining = [m for m in messages if not self._is_system_role_message(m)]
if hoisted:
anthropic_messages_request["messages"] = remaining
system_content = [
block
for source in (
anthropic_messages_request.get("system"),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
]
filtered_system = self._filter_billing_headers_from_system(system_content)
if filtered_system:
anthropic_messages_request["system"] = filtered_system
else:
anthropic_messages_request.pop("system", None)
def get_complete_url(
self,
api_base: Optional[str],

1
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -166,5 +166,6 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
litellm_params=litellm_params,
headers=headers,
)
self._normalize_system_role_messages(anthropic_messages_request, model=model)
self._remove_scope_from_cache_control(anthropic_messages_request)
return anthropic_messages_request

63
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -87,67 +87,6 @@ class AmazonAnthropicClaudeMessagesConfig(
BaseAnthropicMessagesConfig.__init__(self, **kwargs)
AmazonInvokeConfig.__init__(self, **kwargs)
@staticmethod
def _as_system_content_blocks(value: Any) -> list[Any]:
if value is None:
return []
if isinstance(value, list):
return list(value)
if isinstance(value, str):
return [{"type": "text", "text": value}]
return [value]
@staticmethod
def _is_system_role_message(message: Any) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None:
"""Bedrock Invoke validates ``role: "system"`` entries inside ``messages``
per model. Models carrying ``supports_mid_conversation_system`` in the
cost map (the Opus 4.8 family) only reject a leading run ("messages.0:
use the top-level 'system' parameter for the initial system prompt") and
accept mid-conversation entries (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) in place, where they
MUST stay: hoisting one mutates the ``system`` prefix and invalidates the
prompt cache for the entire message history. Older Claude models (Opus
4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position
("role 'system' is not supported on this model"), so without the flag
every system entry is hoisted into the top-level ``system`` field.
Billing-header system blocks are stripped from the top-level ``system``
field regardless of whether anything was hoisted."""
messages = anthropic_messages_request.get("messages")
if not isinstance(messages, list):
return
if _supports_factory(
model=model,
custom_llm_provider="bedrock",
key="supports_mid_conversation_system",
):
leading_count = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
len(messages),
)
hoisted = messages[:leading_count]
remaining = messages[leading_count:]
else:
hoisted = [m for m in messages if self._is_system_role_message(m)]
remaining = [m for m in messages if not self._is_system_role_message(m)]
if hoisted:
anthropic_messages_request["messages"] = remaining
system_content = [
block
for source in (
anthropic_messages_request.get("system"),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
]
filtered_system = self._filter_billing_headers_from_system(system_content)
if filtered_system:
anthropic_messages_request["system"] = filtered_system
else:
anthropic_messages_request.pop("system", None)
def validate_anthropic_messages_environment(
self,
headers: dict,
@ -696,7 +635,7 @@ class AmazonAnthropicClaudeMessagesConfig(
litellm_params=litellm_params,
headers=headers,
)
self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model)
self._normalize_system_role_messages(anthropic_messages_request, model=model)
#########################################################
############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###
#########################################################

2
litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py
View file

@ -142,6 +142,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
headers=headers,
)
self._normalize_system_role_messages(anthropic_messages_request, model=model)
self._remove_scope_from_cache_control(anthropic_messages_request)
anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16"

9
litellm/model_prices_and_context_window_backup.json
View file

@ -2726,6 +2726,7 @@
"supports_max_reasoning_effort": true
},
"azure_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"litellm_provider": "azure_ai",
@ -2756,6 +2757,7 @@
"supports_max_reasoning_effort": true
},
"azure_ai/claude-opus-4-8": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
@ -2828,6 +2830,7 @@
"supports_vision": true
},
"azure_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -36554,6 +36557,7 @@
"prompt_cache_min_tokens": 2048
},
"vertex_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -36584,6 +36588,7 @@
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-fable-5@default": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -36614,6 +36619,7 @@
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-opus-4-8": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -36645,6 +36651,7 @@
"prompt_cache_min_tokens": 1024
},
"vertex_ai/claude-opus-4-8@default": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -36704,6 +36711,7 @@
"prompt_cache_min_tokens": 1024
},
"vertex_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -44237,6 +44245,7 @@
}
},
"vertex_ai/claude-sonnet-5@default": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,

9
model_prices_and_context_window.json
View file

@ -2726,6 +2726,7 @@
"supports_max_reasoning_effort": true
},
"azure_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
"input_cost_per_token": 1e-05,
"output_cost_per_token": 5e-05,
"litellm_provider": "azure_ai",
@ -2756,6 +2757,7 @@
"supports_max_reasoning_effort": true
},
"azure_ai/claude-opus-4-8": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"input_cost_per_token": 5e-06,
"output_cost_per_token": 2.5e-05,
@ -2828,6 +2830,7 @@
"supports_vision": true
},
"azure_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -36645,6 +36648,7 @@
"prompt_cache_min_tokens": 2048
},
"vertex_ai/claude-fable-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -36675,6 +36679,7 @@
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-fable-5@default": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-05,
"cache_creation_input_token_cost_above_1hr": 2e-05,
"cache_read_input_token_cost": 1e-06,
@ -36705,6 +36710,7 @@
"supports_max_reasoning_effort": true
},
"vertex_ai/claude-opus-4-8": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -36736,6 +36742,7 @@
"prompt_cache_min_tokens": 1024
},
"vertex_ai/claude-opus-4-8@default": {
"supports_mid_conversation_system": true,
"supports_adaptive_thinking": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
@ -36795,6 +36802,7 @@
"prompt_cache_min_tokens": 1024
},
"vertex_ai/claude-sonnet-5": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,
@ -44358,6 +44366,7 @@
}
},
"vertex_ai/claude-sonnet-5@default": {
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 2.5e-06,
"cache_creation_input_token_cost_above_1hr": 4e-06,
"cache_read_input_token_cost": 2e-07,

4
tests/e2e/coverage_registry/llm_conversational.yaml
View file

@ -41,6 +41,10 @@
- {id: llm.messages.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Extended thinking via Messages API"}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Flagged Claude 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (#32578/#32831/#32882)", fail_before_fix: proven}
- {id: llm.messages.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py", rationale: "Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (#32831)", fail_before_fix: proven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: proven}
- {id: llm.messages.azure_foundry.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: azure_foundry, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/azure_ai/anthropic/messages_transformation.py", rationale: "Azure Foundry Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: proven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (Kraken Tech RCA gap)", fail_before_fix: unproven}
- {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (Kraken Tech RCA gap)", fail_before_fix: unproven}
- {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
- {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
- {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}

264
tests/e2e/llm_translation/test_messages_mid_conversation_system_native_providers_e2e.py Normal file
View file

@ -0,0 +1,264 @@
"""Live e2e: model-aware mid-conversation ``role: "system"`` handling on the
Azure AI Foundry and Vertex AI ``/v1/messages`` paths.
Azure Foundry and Vertex both serve Claude on the first-party Anthropic Messages
contract, verified live: a mid-conversation ``role: "system"`` reminder is
accepted in place on Claude 4.8+/5 (200) but rejected on Claude 4.7 and older
("role 'system' is not supported on this model", 400), and a *leading* system
entry is rejected on every model ("messages.0: use the top-level 'system'
parameter"). This mirrors Bedrock Invoke (PRs #32578/#32831/#32882); the same
model-gated hoist now runs for these two providers (Kraken Tech RCA gap #3).
Flagged models (``supports_mid_conversation_system`` in the cost map: Claude
4.8+ and the 5 family) must keep the reminder in ``messages`` so the top-level
``system`` prefix stays byte-identical and the prompt cache written on turn one
is read back in full on turn two. Unflagged models (Claude 4.7 and older) must
have the reminder hoisted into the top-level ``system`` field so the call
returns a completion instead of a provider 400.
The conversation shape mirrors what Claude Code sends mid-session: a cached
system prompt, a user turn carrying its own ``cache_control`` breakpoint, a
``role: "system"`` reminder, an assistant turn, and a fresh user turn. The
message-turn breakpoint is what makes the cache assertion able to fail: a cache
entry whose prefix spans ``system`` plus message turns is invalidated when the
reminder is hoisted (the ``system`` field mutates and a turn disappears from
``messages``), while an entry ending at the system block itself would survive
the hoist and mask the regression.
"""
from __future__ import annotations
import time
import pytest
from pydantic import BaseModel
from e2e_config import unique_marker
from e2e_http import Result, unwrap
from endpoints_client import (
CacheControl,
EndpointsClient,
MessagesResult,
RichMessage,
RichMessagesRequest,
TextBlock,
)
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
pytestmark = pytest.mark.e2e
CACHE_PRIMING_DEADLINE_SECONDS = 60.0
CACHE_PRIMING_INTERVAL_SECONDS = 3.0
def _azure_params(model: str) -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=model,
api_base="os.environ/AZURE_AI_API_BASE",
api_key="os.environ/AZURE_AI_API_KEY",
)
def _vertex_params(model: str) -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=model,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="global",
)
def _cacheable_system_block(marker: str) -> TextBlock:
"""A system prompt comfortably above the 1024-token minimum cacheable size,
unique per run so no other run's cache entry can satisfy the read."""
text = " ".join(f"Reference paragraph {index} for run {marker}." for index in range(300))
return TextBlock(text=text, cache_control=CacheControl())
def _user_turn(text: str, *, cached: bool = False) -> RichMessage:
block = TextBlock(text=text, cache_control=CacheControl() if cached else None)
return RichMessage(role="user", content=[block])
def _system_reminder_turn() -> RichMessage:
return RichMessage(
role="system",
content=[TextBlock(text="<system-reminder>Answer with exactly one word.</system-reminder>")],
)
def _post_messages(client: EndpointsClient, key: str, body: RichMessagesRequest) -> Result[MessagesResult]:
return client.gateway.transport.post(
"/v1/messages",
headers=client.gateway.transport.bearer(key),
json=body,
response_type=MessagesResult,
)
def _register_deployment(
client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody
) -> str:
model = f"e2e-midsys-{unique_marker()}"
model_id = client.create_model(model, params)
resources.defer(lambda: client.delete_model(model_id))
return model
def _first_turn_user_text(marker: str) -> str:
"""A first user turn heavy enough (hundreds of tokens) that losing its cache
entry is unambiguous in the usage numbers, unique per attempt so priming
retries never depend on the proxy's response cache behavior."""
notes = " ".join(f"Session note {index} for attempt {marker}." for index in range(100))
return f"Reply with one word.\n{notes}"
class PrimedCache(BaseModel):
first_user_text: str
prefix_read_tokens: int
first_turn_creation_tokens: int
@property
def full_prefix_tokens(self) -> int:
return self.prefix_read_tokens + self.first_turn_creation_tokens
def _prime_prompt_cache(
client: EndpointsClient, key: str, model: str, system_block: TextBlock
) -> PrimedCache:
"""Send first-turn calls (fresh cache-marked user turn each attempt,
identical system prefix) until one both reads the system prefix back from
cache and writes its own user-turn chunk, proving the cache is live in both
directions. Only the pre-reminder turn is ever retried here, so retries can
never warm a mutated-prefix cache entry and mask the regression the second
turn asserts on."""
deadline = time.monotonic() + CACHE_PRIMING_DEADLINE_SECONDS
while True:
user_text = _first_turn_user_text(unique_marker())
body = RichMessagesRequest(
model=model,
system=[system_block],
messages=[_user_turn(user_text, cached=True)],
)
usage = unwrap(_post_messages(client, key, body)).usage
if usage.cache_read_input_tokens > 0 and usage.cache_creation_input_tokens > 0:
return PrimedCache(
first_user_text=user_text,
prefix_read_tokens=usage.cache_read_input_tokens,
first_turn_creation_tokens=usage.cache_creation_input_tokens,
)
if time.monotonic() >= deadline:
pytest.fail(
f"{model}: prompt cache never became readable within "
f"{CACHE_PRIMING_DEADLINE_SECONDS}s (last usage: {usage})"
)
time.sleep(CACHE_PRIMING_INTERVAL_SECONDS)
def _assert_flagged_model_keeps_cache(
client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody
) -> None:
model = _register_deployment(client, resources, params)
key = resources.key(models=[model])
system_block = _cacheable_system_block(unique_marker())
primed = _prime_prompt_cache(client, key, model, system_block)
reminder_turn_body = RichMessagesRequest(
model=model,
system=[system_block],
messages=[
_user_turn(primed.first_user_text, cached=True),
_system_reminder_turn(),
RichMessage(role="assistant", content=[TextBlock(text="OK.")]),
_user_turn("Reply with one word again.", cached=True),
],
)
second = unwrap(_post_messages(client, key, reminder_turn_body))
assert second.text.strip(), f"{model}: reminder turn returned no completion text"
assert second.usage.cache_read_input_tokens >= primed.full_prefix_tokens, (
f"{model}: turn with a mid-conversation system reminder read "
f"{second.usage.cache_read_input_tokens} cached tokens, expected at "
f"least the {primed.full_prefix_tokens} cached on turn one "
f"({primed.prefix_read_tokens} system prefix + "
f"{primed.first_turn_creation_tokens} first user turn); the reminder "
f"was hoisted into the top-level system field, which mutates the cached "
f"prefix and re-bills the conversation at cache-write pricing"
)
def _assert_unflagged_model_hoists_and_succeeds(
client: EndpointsClient, resources: ResourceManager, params: LiteLLMParamsBody
) -> None:
model = _register_deployment(client, resources, params)
key = resources.key(models=[model])
body = RichMessagesRequest(
model=model,
system=[TextBlock(text="You are terse.")],
messages=[
_user_turn(f"Say hi. Run {unique_marker()}."),
_system_reminder_turn(),
RichMessage(role="assistant", content=[TextBlock(text="Hi.")]),
_user_turn("Say bye."),
],
)
completion = unwrap(_post_messages(client, key, body))
assert completion.role == "assistant", f"{model}: unexpected role {completion.role!r}"
assert completion.text.strip(), (
f"{model}: conversation with a mid-conversation system reminder returned "
f"no text; the reminder was forwarded in place to a model that rejects "
f"role 'system' inside messages instead of being hoisted"
)
class TestAzureFoundryMidConversationSystem:
FLAGGED_MODEL = "azure_ai/claude-opus-4-8"
UNFLAGGED_MODEL = "azure_ai/claude-opus-4-7"
@pytest.mark.covers(
"llm.messages.azure_foundry.mid_conversation_system.nonstream.cache_hit",
exercised_on=[],
)
def test_flagged_model_keeps_prompt_cache_across_system_reminder(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
_assert_flagged_model_keeps_cache(endpoints_client, resources, _azure_params(self.FLAGGED_MODEL))
@pytest.mark.covers(
"llm.messages.azure_foundry.mid_conversation_system.nonstream.works",
exercised_on=[],
)
def test_unflagged_model_hoists_system_reminder_and_succeeds(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
_assert_unflagged_model_hoists_and_succeeds(
endpoints_client, resources, _azure_params(self.UNFLAGGED_MODEL)
)
class TestVertexMidConversationSystem:
FLAGGED_MODEL = "vertex_ai/claude-opus-4-8"
UNFLAGGED_MODEL = "vertex_ai/claude-sonnet-4-6"
@pytest.mark.covers(
"llm.messages.vertex.mid_conversation_system.nonstream.cache_hit",
exercised_on=[],
)
def test_flagged_model_keeps_prompt_cache_across_system_reminder(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
_assert_flagged_model_keeps_cache(endpoints_client, resources, _vertex_params(self.FLAGGED_MODEL))
@pytest.mark.covers(
"llm.messages.vertex.mid_conversation_system.nonstream.works",
exercised_on=[],
)
def test_unflagged_model_hoists_system_reminder_and_succeeds(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
_assert_unflagged_model_hoists_and_succeeds(
endpoints_client, resources, _vertex_params(self.UNFLAGGED_MODEL)
)

105
tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py
View file

@ -1,3 +1,5 @@
import copy
import json
import os
import sys
@ -387,3 +389,106 @@ def test_messages_thinking_shape_follows_exact_azure_entry_flag(local_model_cost
assert thinking.get("type") == "enabled"
assert isinstance(thinking.get("budget_tokens"), int)
assert "output_config" not in flipped
def _azure_transform(model, messages, system=None):
config = AzureAnthropicMessagesConfig()
params = {"max_tokens": 256}
if system is not None:
params["system"] = system
return config.transform_anthropic_messages_request(
model=model,
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params=params,
litellm_params=GenericLiteLLMParams(),
headers={},
)
class TestAzureAnthropicMidConversationSystem:
"""Azure AI Foundry serves Claude on the first-party Anthropic /v1/messages
contract: a mid-conversation ``role: "system"`` reminder is accepted in place
on Claude 4.8+/5 but 400s ("role 'system' is not supported on this model") on
older Claude, and a *leading* system entry 400s on every model ("messages.0:
use the top-level 'system' parameter"). These tests pin the model-aware hoist
the config applies so Claude Code sessions neither collapse the prompt cache
on 4.8+ nor hard-fail on 4.7 and older (RCA: Kraken Tech high-spend)."""
def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map):
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = _azure_transform("claude-opus-4-8", messages)
assert result["messages"] == messages
def test_supported_model_hoists_only_leading_system_run(self, local_model_cost_map):
messages = [
{"role": "system", "content": "You are terse."},
{"role": "system", "content": "Cite sources."},
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
result = _azure_transform("claude-opus-4-8", messages)
assert result["messages"] == [
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "You are terse."},
{"type": "text", "text": "Cite sources."},
]
def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map):
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = _azure_transform(
"claude-opus-4-7", messages, system=[{"type": "text", "text": "Base."}]
)
assert result["messages"] == [
{"role": "user", "content": "read the file"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "Base."},
{"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"},
]
def test_azure_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Exact cost-map hits win over the ``claude-mid-conversation-system``
fallback rule, so an ``azure_ai`` Claude 4.8+/5 entry missing the flag would
be treated as unsupported and hoist every reminder, collapsing the prompt
cache. Every mapped azure_ai entry the rule matches must carry the flag."""
import re
import litellm
cost_map_path = os.path.join(
os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json"
)
with open(cost_map_path) as f:
cost_map = json.load(f)
rules = cost_map["fallback_generalizations"]["rules"]
pattern = re.compile(
next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"),
re.IGNORECASE,
)
missing = [
key
for key, info in cost_map.items()
if isinstance(info, dict)
and info.get("litellm_provider") == "azure_ai"
and pattern.search(key)
and info.get("supports_mid_conversation_system") is not True
]
assert missing == []

107
tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
View file

@ -1,3 +1,6 @@
import copy
import json
import os
from unittest.mock import MagicMock, patch
import pytest
@ -565,3 +568,107 @@ def test_messages_thinking_shape_follows_exact_vertex_entry_flag(local_model_cos
assert thinking.get("type") == "enabled"
assert isinstance(thinking.get("budget_tokens"), int)
assert "output_config" not in flipped
def _vertex_transform(model, messages, system=None):
config = VertexAIPartnerModelsAnthropicMessagesConfig()
params = {"max_tokens": 256}
if system is not None:
params["system"] = system
return config.transform_anthropic_messages_request(
model=model,
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params=params,
litellm_params=GenericLiteLLMParams(),
headers={},
)
class TestVertexAnthropicMidConversationSystem:
"""Vertex serves Claude on the first-party Anthropic /v1/messages contract: a
mid-conversation ``role: "system"`` reminder is accepted in place on Claude
4.8+/5 but 400s ("role 'system' is not supported on this model") on older
Claude, and a *leading* system entry 400s on every model ("messages.0: use
the top-level 'system' parameter"). These tests pin the model-aware hoist so
Claude Code sessions neither collapse the prompt cache on 4.8+ nor hard-fail
on 4.7 and older (RCA: Kraken Tech high-spend)."""
def test_supported_model_keeps_mid_conversation_system_in_place(self, local_model_cost_map):
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = _vertex_transform("claude-opus-4-8", messages)
assert result["messages"] == messages
def test_supported_model_hoists_only_leading_system_run(self, local_model_cost_map):
messages = [
{"role": "system", "content": "You are terse."},
{"role": "system", "content": "Cite sources."},
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
result = _vertex_transform("claude-opus-4-8", messages)
assert result["messages"] == [
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "You are terse."},
{"type": "text", "text": "Cite sources."},
]
def test_unsupported_model_hoists_mid_conversation_system(self, local_model_cost_map):
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = _vertex_transform(
"claude-sonnet-4-6", messages, system=[{"type": "text", "text": "Base."}]
)
assert result["messages"] == [
{"role": "user", "content": "read the file"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "Base."},
{"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"},
]
def test_vertex_claude_4_8_plus_cost_map_entries_carry_mid_conversation_system_flag():
"""Exact cost-map hits win over the ``claude-mid-conversation-system``
fallback rule, so a ``vertex_ai`` Claude 4.8+/5 entry missing the flag would
be treated as unsupported and hoist every reminder, collapsing the prompt
cache. Every mapped vertex_ai entry the rule matches must carry the flag."""
import re
import litellm
cost_map_path = os.path.join(
os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json"
)
with open(cost_map_path) as f:
cost_map = json.load(f)
rules = cost_map["fallback_generalizations"]["rules"]
pattern = re.compile(
next(r["pattern"] for r in rules if r["name"] == "claude-mid-conversation-system"),
re.IGNORECASE,
)
missing = [
key
for key, info in cost_map.items()
if isinstance(info, dict)
and str(info.get("litellm_provider", "")).startswith("vertex_ai")
and "claude" in key
and pattern.search(key)
and info.get("supports_mid_conversation_system") is not True
]
assert missing == []