diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py
index 8e5d2cd0a17..14d47a15c6d 100644
--- a/litellm/litellm_core_utils/prompt_templates/common_utils.py
+++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py
@@ -1989,6 +1989,26 @@ def is_encrypted_reasoning_block(block: object) -> bool:
return _carries_encrypted_reasoning(_encrypted_reasoning_field(mapping))
+def is_unsignable_thinking_block(block: object) -> bool:
+ """A thinking block Anthropic cannot accept on input.
+
+ Anthropic verifies the thinking signature cryptographically, so a block whose
+ signature is null, empty, or missing (e.g. from an open-source reasoning model)
+ is rejected with a 400 and must be dropped rather than blanked or repaired, and
+ so is a block whose signature or data carries another provider's encrypted
+ reasoning. A `redacted_thinking` block Anthropic minted is always kept.
+ """
+ if is_encrypted_reasoning_block(block):
+ return True
+ if not isinstance(block, Mapping):
+ return False
+ mapping: Final = cast(Mapping[str, object], block) # cast-ok: narrowed by isinstance
+ if mapping.get("type") != "thinking":
+ return False
+ signature: Final = mapping.get("signature")
+ return not (isinstance(signature, str) and len(signature) > 0)
+
+
def strip_encrypted_reasoning_from_messages(messages: object) -> None:
"""Drop the bridge-tagged reasoning blocks a routed deployment cannot decrypt from
Anthropic-shaped history.
diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py
index 6fc319c26ae..7b12d1e939f 100644
--- a/litellm/litellm_core_utils/prompt_templates/factory.py
+++ b/litellm/litellm_core_utils/prompt_templates/factory.py
@@ -7,7 +7,7 @@ import re
import xml.etree.ElementTree as ET
from collections.abc import Iterator, Mapping, Sequence
from enum import Enum
-from typing import Any, Final, TypedDict, cast, overload
+from typing import Any, Final, TypeAlias, TypedDict, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
@@ -17,6 +17,7 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import anthropic_system_messages
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.types.files import get_file_extension_from_mime_type
@@ -48,8 +49,8 @@ from litellm.types.utils import GenericImageParsingChunk
from .common_utils import (
convert_content_list_to_str,
infer_content_type_from_url_and_content,
- is_encrypted_reasoning_block,
is_non_content_values_set,
+ is_unsignable_thinking_block,
parse_tool_call_arguments,
)
from .image_handling import convert_url_to_base64
@@ -2329,37 +2330,25 @@ def sanitize_messages_for_tool_calling(
return sanitized_messages
-def _is_unsignable_thinking_block(block: object) -> bool:
- """A thinking block that Anthropic cannot accept on input.
-
- Anthropic verifies the thinking signature cryptographically, so a block whose
- signature is null, empty, or missing (e.g. from an open-source reasoning model)
- is rejected with a 400 and must be dropped rather than blanked or repaired, and
- so is a block whose signature or data carries another provider's encrypted
- reasoning. A `redacted_thinking` block Anthropic minted is always kept.
- """
- if is_encrypted_reasoning_block(block):
- return True
- if not isinstance(block, dict) or block.get("type") != "thinking":
- return False
- signature: Final = block.get("signature")
- return not (isinstance(signature, str) and len(signature) > 0)
-
-
def _drop_unsignable_thinking_blocks(
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
) -> list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock]:
- return [block for block in thinking_blocks if not _is_unsignable_thinking_block(block)]
+ return [block for block in thinking_blocks if not is_unsignable_thinking_block(block)]
+
+
+_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
def anthropic_messages_pt(
messages: list[AllMessageValues],
model: str,
llm_provider: str,
-) -> list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]:
+) -> _AnthropicMessageList:
"""
format messages for anthropic
- 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately)
+ 1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately).
+ Models flagged ``supports_mid_conversation_system`` also accept "system" inside
+ messages after a user turn; the caller decides placement, this keeps such messages.
2. The first message always needs to be of role "user"
3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
@@ -2384,7 +2373,7 @@ def anthropic_messages_pt(
# add role=tool support to allow function call result/error submission
user_message_types: Final = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
- new_messages: Final[list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]] = []
+ new_messages: Final[_AnthropicMessageList] = [] # mutable-ok: accumulator behind the mutable return contract
if len(messages) == 0:
if not litellm.modify_params:
@@ -2697,7 +2686,7 @@ def anthropic_messages_pt(
if (
m.get("type", "") == "thinking"
and len(thinking_block) > 0
- and not _is_unsignable_thinking_block(m)
+ and not is_unsignable_thinking_block(m)
): # don't pass empty text blocks. anthropic api raises errors.
anthropic_message: ChatCompletionThinkingBlock | AnthropicMessagesTextParam = cast(
ChatCompletionThinkingBlock, m
@@ -2777,6 +2766,11 @@ def anthropic_messages_pt(
if assistant_content:
new_messages.append({"role": "assistant", "content": assistant_content})
+ ## MID-CONVERSATION SYSTEM MESSAGES (placement is the caller's job) ##
+ while msg_i < len(messages) and messages[msg_i]["role"] == "system":
+ new_messages.extend(anthropic_system_messages(messages[msg_i]))
+ msg_i += 1
+
if msg_i == init_msg_i: # prevent infinite loops
raise litellm.BadRequestError(
message=BAD_MESSAGE_ERROR_STR + f"passed in {messages[msg_i]}",
diff --git a/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
new file mode 100644
index 00000000000..b5e9afca86b
--- /dev/null
+++ b/litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py
@@ -0,0 +1,418 @@
+"""Placement policy for ``role: "system"`` messages that appear after the first turn
+of an Anthropic-shaped chat completions request.
+
+Only the leading run of system messages belongs in the top-level ``system``
+parameter. Hoisting a later one there rewrites the cached prefix, so the provider
+re-bills the whole conversation at cache-write pricing on every reminder (#36559).
+
+Models flagged ``supports_mid_conversation_system`` in the cost map accept the role
+inside ``messages`` under Anthropic's placement rules: the message must directly
+follow a user turn, must be the last entry or be followed by an assistant turn, and
+must not sit next to another system message. OpenAI-shaped clients put system
+messages anywhere, so this module places each run by its neighbours alone: a run
+after a user turn stays with that turn, a run after an assistant turn slides
+behind the user turn that immediately follows it, and a run that ends the array
+or precedes an assistant turn becomes a user turn in place. Runs that land on the
+same slot merge into one system message. No later message can move an earlier
+run, so a client that replays the conversation with more turns appended sends a
+byte-identical prefix and preserved thinking blocks keep their binding.
+
+Models without the flag reject the role inside ``messages``. Their system messages
+become user turns in place, prefixed with an operator note so the model can tell
+the instruction apart from the user's own words. A run caught between a tool call
+and its result moves to just after the result so the ``tool_result`` block stays
+first in the merged user turn.
+
+Every transformation here is a pure function of the message sequence: turn N's
+output stays a prefix of turn N+1's output, which is what keeps the provider-side
+prompt cache readable across turns. Messages are handled in OpenAI format; the
+Anthropic wire shape is built later by ``anthropic_messages_pt``.
+"""
+
+from collections.abc import Iterator, Mapping, Sequence
+from itertools import chain, groupby
+from typing import Final, Literal, TypeAlias
+
+from litellm.types.llms.anthropic import AnthropicMessagesSystemMessageParam, AnthropicSystemMessageContent
+from litellm.types.llms.openai import (
+ AllMessageValues,
+ ChatCompletionCachedContent,
+ ChatCompletionSystemMessage,
+ ChatCompletionTextObject,
+ ChatCompletionUserMessage,
+)
+
+from .common_utils import is_unsignable_thinking_block
+
+CONVERTED_SYSTEM_NOTE: Final = (
+ "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
+)
+
+_USER_TYPE_ROLES: Final = frozenset({"user", "tool", "function"})
+_TOOL_ROLES: Final = frozenset({"tool", "function"})
+_RENDERED_PART_TYPES: Final = frozenset({"text", "image_url", "document", "file"})
+_RENDERED_ASSISTANT_PART_TYPES: Final = frozenset({"text", "server_tool_use"})
+_THINKING_BLOCK_TYPES: Final = frozenset({"thinking", "redacted_thinking"})
+
+_MessageKind: TypeAlias = Literal["system", "tool", "user", "other"]
+_TextPart: TypeAlias = tuple[str, ChatCompletionCachedContent | None]
+
+
+def _as_mapping(value: object) -> Mapping[str, object] | None:
+ return value if isinstance(value, Mapping) else None
+
+
+def parts_of(value: object) -> tuple[object, ...]:
+ return tuple(value) if isinstance(value, Sequence) and not isinstance(value, str) else ()
+
+
+def message_field(message: object, key: str) -> object:
+ """A message field, whether the message is a dict or a pydantic ``Message``.
+
+ Clients replay assistant turns straight from a response, so a history mixes
+ plain dicts with ``litellm.Message`` objects; every predicate reads through here.
+ """
+ mapping: Final = _as_mapping(message)
+ return mapping.get(key) if mapping is not None else getattr(message, key, None)
+
+
+def is_system_message(message: object) -> bool:
+ return message_field(message, "role") == "system"
+
+
+def _is_user_type(message: object) -> bool:
+ return message_field(message, "role") in _USER_TYPE_ROLES
+
+
+def _kind(message: object) -> _MessageKind:
+ role: Final = message_field(message, "role")
+ if role == "system":
+ return "system"
+ if role in _TOOL_ROLES:
+ return "tool"
+ if role == "user":
+ return "user"
+ return "other"
+
+
+def split_leading_system_run(
+ messages: Sequence[AllMessageValues],
+) -> tuple[tuple[AllMessageValues, ...], tuple[AllMessageValues, ...]]:
+ """Split ``messages`` into the leading run of system messages and everything after it."""
+ leading_count: Final = next(
+ (index for index, message in enumerate(messages) if not is_system_message(message)),
+ len(messages),
+ )
+ return tuple(messages[:leading_count]), tuple(messages[leading_count:])
+
+
+def _cache_control(holder: object) -> ChatCompletionCachedContent | None:
+ """The client's ``cache_control`` rebuilt in the only shape Anthropic accepts."""
+ value: Final = _as_mapping(message_field(holder, "cache_control"))
+ if value is None or value.get("type") != "ephemeral":
+ return None
+ ttl: Final = value.get("ttl")
+ if ttl == "1h":
+ one_hour: Final[ChatCompletionCachedContent] = {"type": "ephemeral", "ttl": "1h"}
+ return one_hour
+ if ttl == "5m":
+ five_minutes: Final[ChatCompletionCachedContent] = {"type": "ephemeral", "ttl": "5m"}
+ return five_minutes
+ ephemeral: Final[ChatCompletionCachedContent] = {"type": "ephemeral"}
+ return ephemeral
+
+
+def _text_parts(message: object) -> tuple[_TextPart, ...]:
+ """``(text, cache_control)`` for each non-empty text part of a system message.
+
+ Anthropic rejects empty text blocks and only accepts text in system content. A
+ ``cache_control`` on the message itself belongs to the block built from string
+ content; block-level ``cache_control`` stays with its block.
+ """
+ content: Final = message_field(message, "content")
+ if isinstance(content, str):
+ return ((content, _cache_control(message)),) if content else ()
+ return tuple(part for part in map(_text_part, parts_of(content)) if part is not None)
+
+
+def _text_part(part: object) -> _TextPart | None:
+ if message_field(part, "type") != "text":
+ return None
+ text: Final = message_field(part, "text")
+ return (text, _cache_control(part)) if isinstance(text, str) and text else None
+
+
+def _openai_text_block(part: _TextPart) -> ChatCompletionTextObject:
+ text, cache_control = part
+ if cache_control is None:
+ plain: Final[ChatCompletionTextObject] = {"type": "text", "text": text}
+ return plain
+ cached: Final[ChatCompletionTextObject] = {"type": "text", "text": text, "cache_control": cache_control}
+ return cached
+
+
+def _anthropic_text_block(part: _TextPart) -> AnthropicSystemMessageContent:
+ text, cache_control = part
+ if cache_control is None:
+ plain: Final[AnthropicSystemMessageContent] = {"type": "text", "text": text}
+ return plain
+ cached: Final[AnthropicSystemMessageContent] = {"type": "text", "text": text, "cache_control": cache_control}
+ return cached
+
+
+def anthropic_system_messages(message: object) -> tuple[AnthropicMessagesSystemMessageParam, ...]:
+ """The Anthropic wire message for a system message, or nothing when it carries no text."""
+ blocks: Final = tuple(_anthropic_text_block(part) for part in _text_parts(message))
+ if not blocks:
+ return ()
+ wire: Final[AnthropicMessagesSystemMessageParam] = {
+ "role": "system",
+ "content": list(blocks), # mutable-ok: wire payload; cache_control hooks edit content blocks in place
+ }
+ return (wire,)
+
+
+def system_message_as_user(message: object) -> ChatCompletionUserMessage:
+ """A system message re-rolled as a user turn, prefixed with the operator note."""
+ note: Final[ChatCompletionTextObject] = {"type": "text", "text": CONVERTED_SYSTEM_NOTE}
+ content: Final[list[ChatCompletionTextObject]] = [ # mutable-ok: anthropic_messages_pt only recognises list content
+ note,
+ *(_openai_text_block(part) for part in _text_parts(message)),
+ ]
+ turn: Final[ChatCompletionUserMessage] = {"role": "user", "content": content}
+ return turn
+
+
+def _merged_system_message(run: Sequence[object]) -> tuple[ChatCompletionSystemMessage, ...]:
+ parts: Final = tuple(chain.from_iterable(_text_parts(message) for message in run))
+ if not parts:
+ return ()
+ content: Final[list[ChatCompletionTextObject]] = [ # mutable-ok: anthropic_messages_pt only recognises list content
+ _openai_text_block(part) for part in parts
+ ]
+ merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content}
+ return (merged,)
+
+
+def _converted_user_turns(run: Sequence[object]) -> tuple[ChatCompletionUserMessage, ...]:
+ return tuple(system_message_as_user(message) for message in run if _text_parts(message))
+
+
+def _runs(messages: Sequence[AllMessageValues]) -> tuple[tuple[_MessageKind, tuple[AllMessageValues, ...]], ...]:
+ return tuple((kind, tuple(group)) for kind, group in groupby(messages, key=_kind))
+
+
+def _converted_for_unflagged_model(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
+ """Convert every system message to a user turn in place.
+
+ A system run whose follower is a tool message is emitted after that tool run:
+ ``tool_result`` blocks have to open the merged user turn.
+ """
+ runs: Final = _runs(messages)
+
+ def emit(index: int) -> tuple[AllMessageValues, ...]:
+ kind, run = runs[index]
+ follower: Final = runs[index + 1][0] if index + 1 < len(runs) else None
+ if kind == "system":
+ return () if follower == "tool" else _converted_user_turns(run)
+ if kind == "tool" and index > 0 and runs[index - 1][0] == "system":
+ return (*run, *_converted_user_turns(runs[index - 1][1]))
+ return run
+
+ return tuple(chain.from_iterable(emit(index) for index in range(len(runs))))
+
+
+def _user_type_blocks(messages: Sequence[AllMessageValues]) -> tuple[tuple[bool, tuple[int, ...]], ...]:
+ """Maximal groups of consecutive non-system messages, keyed by whether they are user-type.
+
+ Consecutive user-type messages become one user turn on the wire, so a group is
+ the unit a system message can validly follow.
+ """
+ indexed: Final = tuple((index, message) for index, message in enumerate(messages) if not is_system_message(message))
+ return tuple(
+ (is_user, tuple(index for index, _ in group))
+ for is_user, group in groupby(indexed, key=lambda pair: _is_user_type(pair[1]))
+ )
+
+
+def _system_runs(messages: Sequence[AllMessageValues]) -> tuple[tuple[int, ...], ...]:
+ """Index runs of consecutive system messages."""
+ system_indices: Final = tuple(index for index, message in enumerate(messages) if is_system_message(message))
+ return tuple(
+ tuple(index for _, index in group)
+ for _, group in groupby(enumerate(system_indices), key=lambda pair: pair[1] - pair[0])
+ )
+
+
+def _block_containing(message_index: int, blocks: Sequence[tuple[bool, tuple[int, ...]]]) -> int:
+ return next(index for index, (_, indices) in enumerate(blocks) if message_index in indices)
+
+
+def _thinking_block_renders(block: object) -> bool:
+ """A thinking block the converter keeps: one Anthropic can verify, so never bridged encrypted reasoning."""
+ return message_field(block, "type") in _THINKING_BLOCK_TYPES and not is_unsignable_thinking_block(block)
+
+
+def _assistant_part_renders(part: object) -> bool:
+ """A text part always renders: the converter pads empty text with a placeholder."""
+ part_type: Final = message_field(part, "type")
+ if part_type == "thinking":
+ thinking: Final = message_field(part, "thinking")
+ return isinstance(thinking, str) and bool(thinking) and _thinking_block_renders(part)
+ return part_type in _RENDERED_ASSISTANT_PART_TYPES or (
+ isinstance(part_type, str) and part_type.endswith("_tool_result")
+ )
+
+
+def _separate_thinking_blocks_render(message: object, parts: Sequence[object]) -> bool:
+ """``thinking_blocks`` reach the wire only when no inline thinking part claims the slot.
+
+ The converter skips the separate blocks as soon as the content list carries a
+ ``thinking`` or ``redacted_thinking`` part, whether or not that part itself renders.
+ """
+ if any(message_field(part, "type") in _THINKING_BLOCK_TYPES for part in parts):
+ return False
+ return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
+
+
+def _assistant_renders(message: object) -> bool:
+ """Whether ``anthropic_messages_pt`` puts a block on the wire for this assistant message.
+
+ String content (the converter pads an empty one with a placeholder), a text part,
+ a signed thinking part, a server tool part, tool calls, a function call, a kept
+ thinking block and compaction blocks each render. An assistant message with none
+ of them, such as ``content: None`` or an empty list, vanishes from the wire.
+ """
+ content: Final = message_field(message, "content")
+ if isinstance(content, str):
+ return True
+ parts: Final = parts_of(content)
+ return (
+ any(_assistant_part_renders(part) for part in parts)
+ or _separate_thinking_blocks_render(message, parts)
+ or bool(message_field(message, "tool_calls"))
+ or bool(message_field(message, "function_call"))
+ or bool(message_field(message_field(message, "provider_specific_fields"), "compaction_blocks"))
+ )
+
+
+def _renders(message: object) -> bool:
+ """Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
+
+ A tool message always becomes a ``tool_result`` and a user message with string
+ content always becomes a text block (empty text gets a placeholder). A user list
+ renders only through parts of a type the converter emits; ``None``, an empty list,
+ and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
+ """
+ role: Final = message_field(message, "role")
+ if role in _TOOL_ROLES:
+ return True
+ if role == "assistant":
+ return _assistant_renders(message)
+ content: Final = message_field(message, "content")
+ return isinstance(content, str) or any(
+ message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
+ )
+
+
+def _rendered_block(
+ message_index: int,
+ messages: Sequence[AllMessageValues],
+ blocks: Sequence[tuple[bool, tuple[int, ...]]],
+) -> int | None:
+ block_index: Final = _block_containing(message_index, blocks)
+ _, indices = blocks[block_index]
+ return block_index if any(_renders(messages[index]) for index in indices) else None
+
+
+def _system_may_follow(
+ block_index: int,
+ messages: Sequence[AllMessageValues],
+ blocks: Sequence[tuple[bool, tuple[int, ...]]],
+) -> bool:
+ """Whether a system message behind this block precedes an assistant turn or ends the array on the wire.
+
+ Blocks alternate between user-type and assistant, so the check is whether the
+ first later block that puts anything on the wire is an assistant block.
+ """
+ return next(
+ (
+ not is_user
+ for is_user, indices in blocks[block_index + 1 :]
+ if any(_renders(messages[index]) for index in indices)
+ ),
+ True,
+ )
+
+
+def _anchor_block(
+ run: Sequence[int],
+ messages: Sequence[AllMessageValues],
+ blocks: Sequence[tuple[bool, tuple[int, ...]]],
+) -> int | None:
+ """The user-type block a system run must follow, or ``None`` when it converts in place.
+
+ The run never starts at 0: the leading system run was split off before this
+ policy runs, so the message before a run is always a non-system message. Only
+ the run's neighbours decide, so a request that replays these messages with more
+ turns appended places the run identically. A block that puts nothing on the wire
+ cannot anchor a run: the system message would land first or behind an assistant
+ turn, so the run converts in place instead. The same happens when the assistant
+ turn after the anchor puts nothing on the wire and a user turn follows it: the
+ system message would sit directly before that user turn, which Anthropic rejects.
+ """
+ previous: Final = run[0] - 1
+ neighbour: Final = previous if _is_user_type(messages[previous]) else run[-1] + 1
+ if neighbour >= len(messages) or not _is_user_type(messages[neighbour]):
+ return None
+ block_index: Final = _rendered_block(neighbour, messages, blocks)
+ if block_index is None or not _system_may_follow(block_index, messages, blocks):
+ return None
+ return block_index
+
+
+def _placed_for_flagged_model(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
+ """Keep system messages as ``role: "system"`` at a placement Anthropic accepts.
+
+ A run already sitting after a user-type message stays with that user turn. A
+ run after an assistant turn moves behind the user turn that immediately follows
+ it. A run that ends the array or is followed by an assistant turn becomes user
+ turns in place, so replaying the same messages with more turns appended cannot
+ move it. Runs that share a user turn merge into one system message.
+ """
+ blocks: Final = _user_type_blocks(messages)
+ anchors: Final = tuple((run, _anchor_block(run, messages, blocks)) for run in _system_runs(messages))
+
+ def messages_of(run: tuple[int, ...]) -> tuple[AllMessageValues, ...]:
+ return tuple(messages[index] for index in run)
+
+ def anchored_to(block_index: int) -> tuple[AllMessageValues, ...]:
+ anchored_runs: Final = tuple(run for run, anchor in anchors if anchor == block_index)
+ return tuple(chain.from_iterable(map(messages_of, anchored_runs)))
+
+ def converted_after(message_index: int) -> tuple[ChatCompletionUserMessage, ...]:
+ following_runs: Final = tuple(run for run, anchor in anchors if anchor is None and run[0] == message_index + 1)
+ return tuple(chain.from_iterable(_converted_user_turns(messages_of(run)) for run in following_runs))
+
+ def emit(block_index: int) -> Iterator[AllMessageValues]:
+ is_user, indices = blocks[block_index]
+ for index in indices:
+ yield messages[index]
+ yield from converted_after(index)
+ if is_user:
+ yield from _merged_system_message(anchored_to(block_index))
+
+ return tuple(chain.from_iterable(emit(block_index) for block_index in range(len(blocks))))
+
+
+def place_mid_conversation_system(
+ messages: Sequence[AllMessageValues],
+ *,
+ supports_mid_conversation_system: bool,
+) -> tuple[AllMessageValues, ...]:
+ """Apply the placement policy to the messages after the leading system run."""
+ if not any(is_system_message(message) for message in messages):
+ return tuple(messages)
+ if supports_mid_conversation_system:
+ return _placed_for_flagged_model(messages)
+ return _converted_for_unflagged_model(messages)
diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py
index b7c2ce3c568..3bffee48d6a 100644
--- a/litellm/llms/anthropic/chat/transformation.py
+++ b/litellm/llms/anthropic/chat/transformation.py
@@ -31,13 +31,17 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
async_inline_remote_media,
inline_remote_image_urls,
)
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+ place_mid_conversation_system,
+ split_leading_system_run,
+)
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.types.llms.anthropic import (
ANTHROPIC_ADVISOR_TOOL_TYPE,
ANTHROPIC_BETA_HEADER_VALUES,
ANTHROPIC_HOSTED_TOOLS,
- AllAnthropicMessageValues,
+ AllAnthropicPassThroughMessageValues,
AllAnthropicToolsValues,
AnthropicCodeExecutionTool,
AnthropicComputerTool,
@@ -87,6 +91,7 @@ from litellm.utils import (
get_max_tokens,
has_tool_call_blocks,
last_assistant_with_tool_calls_has_no_thinking_blocks,
+ supports_mid_conversation_system,
supports_reasoning,
token_counter,
)
@@ -1743,10 +1748,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
def add_code_execution_tool(
self,
- messages: list[AllAnthropicMessageValues],
+ messages: list[AllAnthropicPassThroughMessageValues],
tools: list[AllAnthropicToolsValues | dict],
) -> list[AllAnthropicToolsValues | dict]:
- """if 'container_upload' in messages, add code_execution tool"""
+ """if 'container_upload' in messages, add code_execution tool
+
+ Takes the pass-through union because the translator emits ``role: "system"``
+ in ``messages`` for models that accept it; only ``content`` is read here."""
add_code_execution_tool = False
for message in messages:
message_content = message.get("content", None)
@@ -1966,16 +1974,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
- # Separate system prompt from rest of message
- anthropic_system_message_list: Final = self.translate_system_message(messages=messages)
+ # Only the leading system run becomes the top-level system prompt. A later
+ # system message stays in the conversation: hoisting it rewrites the cached
+ # prefix and re-bills the whole history at cache-write pricing (#36559).
+ leading_system_run, later_messages = split_leading_system_run(messages)
+ anthropic_system_message_list: Final = self.translate_system_message(
+ messages=list(leading_system_run) # mutable-ok: translate_system_message pops from the list it is given
+ )
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
optional_params["system"] = anthropic_system_message_list
+ conversation: Final = place_mid_conversation_system(
+ later_messages,
+ supports_mid_conversation_system=supports_mid_conversation_system(
+ model=model, custom_llm_provider=self.custom_llm_provider
+ ),
+ )
# Format rest of message according to anthropic guidelines
try:
anthropic_messages = anthropic_messages_pt(
model=model,
- messages=messages,
+ messages=list(conversation), # mutable-ok: anthropic_messages_pt rewrites entries in place
llm_provider=self._resolved_provider,
)
except Exception as e:
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py b/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py
index ddefec6bac9..f588133812c 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/mid_conversation_system.py
@@ -2,9 +2,7 @@ from collections.abc import Mapping, Sequence
from itertools import groupby
from typing import Final
-CONVERTED_SYSTEM_NOTE: Final = (
- "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
-)
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE
def as_system_content_blocks(value: object) -> list[object]:
diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py
index 497020c2836..2a2f3052b2a 100644
--- a/litellm/llms/bedrock/chat/converse_transformation.py
+++ b/litellm/llms/bedrock/chat/converse_transformation.py
@@ -7,7 +7,8 @@ import json
import re
import time
import types
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
+from itertools import chain
from typing import TYPE_CHECKING, Final, Literal, cast, overload
import httpx
@@ -34,6 +35,12 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_tools_pt,
make_valid_bedrock_tool_name,
)
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+ CONVERTED_SYSTEM_NOTE,
+ is_system_message,
+ message_field,
+ parts_of,
+)
from litellm.llms.anthropic.chat.transformation import (
DROP_UNSUPPORTED_ADAPTIVE_THINKING_WARNING,
DROP_UNSUPPORTED_OUTPUT_CONFIG_WARNING,
@@ -55,9 +62,11 @@ from litellm.types.llms.openai import (
ChatCompletionAnnotation,
ChatCompletionAssistantMessage,
ChatCompletionAssistantToolCall,
+ ChatCompletionCachedContent,
ChatCompletionRedactedThinkingBlock,
ChatCompletionResponseMessage,
ChatCompletionSystemMessage,
+ ChatCompletionTextObject,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
@@ -1343,30 +1352,157 @@ class AmazonConverseConfig(BaseConfig):
cache_point["ttl"] = ttl
return cache_point
+ @staticmethod
+ def _assistant_has_tool_calls(message: object) -> bool:
+ return message_field(message, "role") == "assistant" and bool(message_field(message, "tool_calls"))
+
+ @staticmethod
+ def _opens_with_tool_result(message: object) -> bool:
+ """Whether the message starts a tool-result turn on Converse.
+
+ ``_bedrock_converse_messages_pt`` builds ``toolResult`` blocks from ``tool``
+ messages only, so a ``function`` message never opens one."""
+ role: Final = message_field(message, "role")
+ if role == "tool":
+ return True
+ if role != "user":
+ return False
+ first_part: Final = next(iter(parts_of(message_field(message, "content"))), None)
+ return message_field(first_part, "type") == "tool_result"
+
+ def _system_run_before(self, messages: Sequence[AllMessageValues], index: int) -> Sequence[AllMessageValues]:
+ start: Final = next(
+ (j + 1 for j in range(index - 1, -1, -1) if not is_system_message(messages[j])),
+ 0,
+ )
+ return messages[start:index]
+
+ def _system_run_end(self, messages: Sequence[AllMessageValues], index: int) -> int:
+ return next(
+ (j for j in range(index, len(messages)) if not is_system_message(messages[j])),
+ len(messages),
+ )
+
+ def _reordered_around_tool_results(
+ self, messages: Sequence[AllMessageValues], index: int
+ ) -> tuple[AllMessageValues, ...]:
+ """Move a system run wedged between an assistant tool-call turn and its
+ tool-result turn(s) to after the tool results.
+
+ A converted system entry becomes a user turn, and a user turn between
+ a tool call and its result would split them. Everything else stays in
+ place so the cached prefix stays byte-identical."""
+ message: Final = messages[index]
+ if self._opens_with_tool_result(message):
+ if index + 1 < len(messages) and self._opens_with_tool_result(messages[index + 1]):
+ return (message,)
+ tool_run_start: Final = next(
+ (j + 1 for j in range(index, -1, -1) if not self._opens_with_tool_result(messages[j])),
+ 0,
+ )
+ run: Final = self._system_run_before(messages, tool_run_start)
+ prev_idx: Final = tool_run_start - len(run) - 1
+ if run and prev_idx >= 0 and self._assistant_has_tool_calls(messages[prev_idx]):
+ return (message, *run)
+ return (message,)
+ if not is_system_message(message):
+ return (message,)
+ run_start: Final = next(
+ (j + 1 for j in range(index - 1, -1, -1) if not is_system_message(messages[j])),
+ 0,
+ )
+ run_end: Final = self._system_run_end(messages, index)
+ follower: Final = messages[run_end] if run_end < len(messages) else None
+ if (
+ follower is not None
+ and self._opens_with_tool_result(follower)
+ and run_start > 0
+ and self._assistant_has_tool_calls(messages[run_start - 1])
+ ):
+ return ()
+ return (message,)
+
+ def _system_role_message_as_user(self, message: ChatCompletionSystemMessage) -> ChatCompletionUserMessage | None:
+ """Convert a mid-conversation system entry to a user turn, in place.
+
+ The Converse API only accepts user/assistant roles in ``messages``,
+ so keeping the role is not an option. Hoisting it to the top-level
+ ``system`` block would mutate the system prefix and collapse implicit
+ prompt caching; converting in place keeps everything before the entry
+ byte-identical. An entry that carries no text becomes ``None``."""
+ text_blocks: Final = self._converted_text_blocks(message)
+ if not text_blocks:
+ return None
+ note: Final = ChatCompletionTextObject(type="text", text=CONVERTED_SYSTEM_NOTE)
+ body: Final = [ # mutable-ok: _bedrock_converse_messages_pt narrows content with isinstance(list)
+ note,
+ *text_blocks,
+ ]
+ return ChatCompletionUserMessage(role="user", content=body)
+
+ def _converted_or_kept(self, message: AllMessageValues) -> AllMessageValues | None:
+ if not is_system_message(message):
+ return message
+ return self._system_role_message_as_user(
+ cast(ChatCompletionSystemMessage, message) # cast-ok: the role is checked on the line above
+ )
+
+ def _converted_text_blocks(self, message: ChatCompletionSystemMessage) -> tuple[ChatCompletionTextObject, ...]:
+ content: Final = message["content"]
+ if isinstance(content, str):
+ return (self._converted_text_block(content, message.get("cache_control")),) if content else ()
+ parts: Final[Sequence[object]] = content or ()
+ return tuple(
+ self._converted_text_block(part["text"], part.get("cache_control"))
+ for part in map(self._text_part, parts)
+ if part is not None
+ )
+
+ @staticmethod
+ def _text_part(part: object) -> ChatCompletionTextObject | None:
+ if not isinstance(part, dict) or part.get("type") != "text" or not part.get("text"):
+ return None
+ return cast(ChatCompletionTextObject, part) # cast-ok: the shape is checked on the line above
+
+ @staticmethod
+ def _converted_text_block(text: str, cache_control: ChatCompletionCachedContent | None) -> ChatCompletionTextObject:
+ if cache_control is None:
+ return ChatCompletionTextObject(type="text", text=text)
+ return ChatCompletionTextObject(type="text", text=text, cache_control=cache_control)
+
def _transform_system_message(
self, messages: list[AllMessageValues], model: str | None = None
) -> tuple[list[AllMessageValues], list[SystemContentBlock]]:
- system_prompt_indices: Final = []
+ leading_count: Final = next(
+ (i for i, m in enumerate(messages) if not is_system_message(m)),
+ len(messages),
+ )
+ hoisted: Final = messages[:leading_count]
+ remaining: Final = messages[leading_count:]
system_content_blocks: Final[list[SystemContentBlock]] = []
- for idx, message in enumerate(messages):
- if message["role"] == "system":
- system_prompt_indices.append(idx)
- if isinstance(message["content"], str) and message["content"]:
- system_content_blocks.append(SystemContentBlock(text=message["content"]))
- cache_block = self.get_cache_point_block(message, block_type="system", model=model)
- if cache_block:
- system_content_blocks.append(cache_block)
- elif isinstance(message["content"], list):
- for m in message["content"]:
- if m.get("type") == "text" and m.get("text"):
- system_content_blocks.append(SystemContentBlock(text=m["text"]))
- cache_block = self.get_cache_point_block(m, block_type="system", model=model)
- if cache_block:
- system_content_blocks.append(cache_block)
- if len(system_prompt_indices) > 0:
- for idx in reversed(system_prompt_indices):
- messages.pop(idx)
- return messages, system_content_blocks
+ for message in hoisted:
+ if message["role"] != "system":
+ continue
+ if isinstance(message["content"], str) and message["content"]:
+ system_content_blocks.append(SystemContentBlock(text=message["content"]))
+ cache_block = self.get_cache_point_block(message, block_type="system", model=model)
+ if cache_block:
+ system_content_blocks.append(cache_block)
+ elif isinstance(message["content"], list):
+ for m in message["content"]:
+ if m.get("type") == "text" and m.get("text"):
+ system_content_blocks.append(SystemContentBlock(text=m["text"]))
+ cache_block = self.get_cache_point_block(m, block_type="system", model=model)
+ if cache_block:
+ system_content_blocks.append(cache_block)
+ reordered: Final = tuple(
+ chain.from_iterable(
+ self._reordered_around_tool_results(remaining, index) for index in range(len(remaining))
+ )
+ )
+ converted: Final = tuple(self._converted_or_kept(message) for message in reordered)
+ kept: Final = [message for message in converted if message is not None] # mutable-ok: converse pt takes a list
+ return kept, system_content_blocks
def _transform_inference_params(self, inference_params: dict) -> InferenceConfig:
if "top_k" in inference_params:
diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py
index 042df6f37fa..a818daf554d 100644
--- a/litellm/types/llms/anthropic.py
+++ b/litellm/types/llms/anthropic.py
@@ -393,7 +393,8 @@ class AnthropicMessagesSystemMessageParam(TypedDict, total=False):
AllAnthropicMessageValues = AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam
-# System is not a native Anthropic message role; only pass-through adapters use this union.
+# role=system inside messages is accepted after a user turn on models flagged
+# supports_mid_conversation_system; pass-through adapters and the chat translator both emit it.
AllAnthropicPassThroughMessageValues: TypeAlias = (
AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam | AnthropicMessagesSystemMessageParam
)
diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml
index e4e1ac2c7b6..6fa9991705c 100644
--- a/tests/e2e/coverage_registry/llm_conversational.yaml
+++ b/tests/e2e/coverage_registry/llm_conversational.yaml
@@ -23,6 +23,8 @@
- {id: llm.chat_completions.anthropic.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude prompt caching"}
- {id: llm.chat_completions.anthropic.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: anthropic, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude extended thinking"}
- {id: llm.chat_completions.anthropic.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: anthropic, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Claude response_schema"}
+- {id: llm.chat_completions.anthropic.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to first-party Anthropic: flagged Claude 4.8+/5 must keep a mid-conversation role system reminder in messages; hoisting it into the top-level system field mutates the cached prefix and re-bills the conversation at cache-write pricing (#36559)", fail_before_fix: proven}
+- {id: llm.chat_completions.anthropic.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: anthropic, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to first-party Anthropic: Claude <= 4.7 and Haiku 4.5 reject role system inside messages, so unflagged models must convert a mid-conversation reminder to a user turn in place (hoisting collapses the prompt cache) and still answer (#36559)", fail_before_fix: proven}
- {id: llm.chat_completions.bedrock_converse.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Bedrock Converse unified"}
- {id: llm.chat_completions.bedrock_converse.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming over Converse"}
- {id: llm.chat_completions.bedrock_converse.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Converse function_calling; AWS adoption"}
@@ -34,6 +36,8 @@
- {id: llm.chat_completions.bedrock_converse.batch_deployment.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: batch_deployment, streaming: nonstream, assertions: [works], source: "types/utils.py bedrock_batch_litellm_params", rationale: "A deployment carrying the documented batch-only S3 keys (s3_access_key_id, s3_secret_access_key, s3_encryption_key_id) must still serve ordinary chat; unregistered keys fall into optional_params and are forwarded as additionalModelRequestFields, which Bedrock 400s and which puts the S3 secret in the request body and debug log (LIT-8290)", fail_before_fix: proven}
- {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"}
- {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"}
+- {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke builds the Anthropic request through AnthropicConfig.transform_request, so flagged Claude 4.8+/5 must keep a mid-conversation role system reminder in messages; hoisting mutates the cached prefix and collapses the prompt cache (#36559)", fail_before_fix: proven}
+- {id: llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/anthropic/chat/transformation.py", rationale: "OpenAI-format chat to Bedrock Invoke: Claude <= 4.7 and Haiku 4.5 reject role system inside messages, so unflagged models must convert a mid-conversation reminder to a user turn in place (hoisting collapses the prompt cache) and still answer (#36559)", fail_before_fix: proven}
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
diff --git a/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py b/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py
new file mode 100644
index 00000000000..480225b502e
--- /dev/null
+++ b/tests/e2e/llm_translation/test_chat_mid_conversation_system_e2e.py
@@ -0,0 +1,322 @@
+"""Live e2e: mid-conversation ``role: "system"`` handling on the OpenAI-format
+/v1/chat/completions path is model-aware for first-party Anthropic and Bedrock
+Invoke, both of which build the Anthropic request through
+``AnthropicConfig.transform_request`` (#36559).
+
+Only the leading run of system messages becomes the top-level ``system``
+parameter. A ``role: "system"`` entry that appears later in ``messages`` used to
+be hoisted into that same field, which rewrote the cached prefix and re-billed
+the whole conversation at cache-write pricing on every reminder. Models flagged
+``supports_mid_conversation_system`` in the cost map (Claude 4.8+ and the 5
+family) must keep the reminder in ``messages`` as ``role: "system"``; models
+without the flag (Claude 4.7 and older, Haiku 4.5) reject that role inside
+``messages``, so the proxy must convert the reminder to a user turn in place,
+prefixed with an operator note. Either way the prompt cache written on turn one
+must be read back in full on turn two.
+
+The conversation shape mirrors what an OpenAI-SDK client sends mid-session: a
+cached system prompt, a user turn carrying its own ``cache_control`` breakpoint,
+an assistant turn, a ``role: "system"`` reminder, and a fresh user turn. The
+message-turn breakpoint is what makes the cache assertion able to fail: a cache
+entry whose prefix spans ``system`` plus message turns is invalidated when the
+reminder is hoisted (the ``system`` field mutates and a turn disappears from
+``messages``), while an entry ending at the system block itself would survive
+the hoist and mask the regression.
+
+The provider-native ``cache_control`` request shape is not expressible with the
+shared ``ChatBody`` (whose content parts carry no cache_control), so the body is
+built from the typed content blocks shared in ``models.py``.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+from pydantic import BaseModel
+
+from e2e_config import unique_marker
+from e2e_http import Result, unwrap
+from lifecycle import ResourceManager
+from models import CacheControl, ChatResponse, LiteLLMParamsBody, RichMessage, TextBlock, Usage
+from passthrough_client import PassthroughClient
+
+pytestmark = [pytest.mark.e2e, pytest.mark.provider_live]
+
+CACHE_PRIMING_DEADLINE_SECONDS = 60.0
+CACHE_PRIMING_INTERVAL_SECONDS = 3.0
+CACHE_WARM_CONSECUTIVE_READS = 3
+
+
+class CacheChatRequest(BaseModel):
+ """OpenAI-format chat body whose content blocks carry ``cache_control``."""
+
+ model: str
+ messages: list[RichMessage]
+ max_tokens: int = 64
+ cache: dict[str, bool] = {"no-cache": True}
+
+
+def _anthropic_params(model: str) -> LiteLLMParamsBody:
+ return LiteLLMParamsBody(model=model, api_key="os.environ/ANTHROPIC_API_KEY")
+
+
+def _invoke_params(model: str, region: str) -> LiteLLMParamsBody:
+ return LiteLLMParamsBody(model=model, aws_region_name=region)
+
+
+def _cacheable_system_turn(marker: str) -> RichMessage:
+ """A system prompt comfortably above the 4096-token minimum cacheable size
+ of Haiku 4.5 (the smallest model here), unique per run so no other run's
+ cache entry can satisfy the read."""
+ text = " ".join(f"Reference paragraph {index} for run {marker}." for index in range(300))
+ return RichMessage(role="system", content=[TextBlock(text=text, cache_control=CacheControl())])
+
+
+def _user_turn(text: str, *, cached: bool = False) -> RichMessage:
+ block = TextBlock(text=text, cache_control=CacheControl() if cached else None)
+ return RichMessage(role="user", content=[block])
+
+
+def _assistant_turn(text: str) -> RichMessage:
+ return RichMessage(role="assistant", content=[TextBlock(text=text)])
+
+
+def _system_reminder_turn() -> RichMessage:
+ return RichMessage(
+ role="system",
+ content=[TextBlock(text="Answer with exactly one word.")],
+ )
+
+
+def _post_chat(client: PassthroughClient, key: str, body: CacheChatRequest) -> Result[ChatResponse]:
+ return client.proxy.transport.post(
+ "/v1/chat/completions",
+ headers=client.proxy.transport.bearer(key),
+ json=body,
+ response_type=ChatResponse,
+ )
+
+
+def _register_deployment(client: PassthroughClient, resources: ResourceManager, params: LiteLLMParamsBody) -> str:
+ model = f"e2e-chat-midsys-{unique_marker()}"
+ model_id = client.proxy.create_model(model, params)
+ resources.defer(lambda: client.proxy.delete_model(model_id))
+ return model
+
+
+def _first_turn_user_text(marker: str) -> str:
+ """A first user turn heavy enough (hundreds of tokens) that losing its cache
+ entry is unambiguous in the usage numbers, unique per attempt so priming
+ retries never depend on the proxy's response cache behavior."""
+ notes = " ".join(f"Session note {index} for attempt {marker}." for index in range(100))
+ return f"Reply with one word.\n{notes}"
+
+
+def _cache_read_tokens(usage: Usage | None) -> int:
+ """Cache-read tokens however the chat usage reports them: the Anthropic-style
+ ``cache_read_input_tokens`` litellm forwards, or the OpenAI-style
+ ``prompt_tokens_details.cached_tokens`` it mirrors them into."""
+ if usage is None:
+ return 0
+ if usage.cache_read_input_tokens:
+ return usage.cache_read_input_tokens
+ if usage.prompt_tokens_details and usage.prompt_tokens_details.cached_tokens:
+ return usage.prompt_tokens_details.cached_tokens
+ return 0
+
+
+def _cache_creation_tokens(usage: Usage | None) -> int:
+ if usage is None:
+ return 0
+ return usage.cache_creation_input_tokens or 0
+
+
+def _response_text(response: ChatResponse) -> str:
+ return "".join(choice.message.content or "" for choice in response.choices if choice.message)
+
+
+def _response_role(response: ChatResponse) -> str | None:
+ first = response.choices[0].message if response.choices else None
+ return first.role if first else None
+
+
+class PrimedCache(BaseModel):
+ first_user_text: str
+ prefix_read_tokens: int
+ first_turn_creation_tokens: int
+
+ @property
+ def full_prefix_tokens(self) -> int:
+ return self.prefix_read_tokens + self.first_turn_creation_tokens
+
+
+def _prime_prompt_cache(client: PassthroughClient, key: str, model: str, system_turn: RichMessage) -> PrimedCache:
+ """Send first-turn calls (fresh cache-marked user turn each attempt,
+ identical system prefix) until one both reads the system prefix back from
+ cache and writes its own user-turn chunk, then re-send that exact turn until
+ its own chunk reads back on three sends in a row, proving the cache is live
+ in both directions before the reminder turn goes out (a freshly written entry
+ can take a few seconds to become readable). Only the pre-reminder turn is
+ ever retried here, so retries can never warm a mutated-prefix cache entry and
+ mask the regression the second turn asserts on."""
+ deadline = time.monotonic() + CACHE_PRIMING_DEADLINE_SECONDS
+ while True:
+ user_text = _first_turn_user_text(unique_marker())
+ body = CacheChatRequest(model=model, messages=[system_turn, _user_turn(user_text, cached=True)])
+ usage = unwrap(_post_chat(client, key, body)).usage
+ read_tokens = _cache_read_tokens(usage)
+ creation_tokens = _cache_creation_tokens(usage)
+ if read_tokens > 0 and creation_tokens > 0:
+ primed = PrimedCache(
+ first_user_text=user_text,
+ prefix_read_tokens=read_tokens,
+ first_turn_creation_tokens=creation_tokens,
+ )
+ if _first_turn_reads_back(client, key, body, primed.full_prefix_tokens, deadline):
+ return primed
+ if time.monotonic() >= deadline:
+ pytest.fail(
+ f"{model}: prompt cache never became readable in full within "
+ f"{CACHE_PRIMING_DEADLINE_SECONDS}s (last usage: {usage})"
+ )
+ time.sleep(CACHE_PRIMING_INTERVAL_SECONDS)
+
+
+def _reads_full_prefix(client: PassthroughClient, key: str, body: CacheChatRequest, full_prefix_tokens: int) -> bool:
+ return _cache_read_tokens(unwrap(_post_chat(client, key, body)).usage) >= full_prefix_tokens
+
+
+def _first_turn_reads_back(
+ client: PassthroughClient,
+ key: str,
+ body: CacheChatRequest,
+ full_prefix_tokens: int,
+ deadline: float,
+) -> bool:
+ """True once the full prefix reads back on CACHE_WARM_CONSECUTIVE_READS sends in
+ a row. Some providers' global endpoints serve the prompt cache per region, so a
+ fresh entry can be missing from the region the next request lands on; each miss
+ re-creates the entry there, so the streak converges as the regions warm up."""
+ while time.monotonic() < deadline:
+ if all(_reads_full_prefix(client, key, body, full_prefix_tokens) for _ in range(CACHE_WARM_CONSECUTIVE_READS)):
+ return True
+ time.sleep(CACHE_PRIMING_INTERVAL_SECONDS)
+ return False
+
+
+def _reminder_turn_body(model: str, system_turn: RichMessage, primed: PrimedCache) -> CacheChatRequest:
+ """Turn two in OpenAI shape: the primed prefix, an assistant reply, the
+ mid-conversation system reminder, and a fresh cache-marked user turn."""
+ return CacheChatRequest(
+ model=model,
+ messages=[
+ system_turn,
+ _user_turn(primed.first_user_text, cached=True),
+ _assistant_turn("OK."),
+ _system_reminder_turn(),
+ _user_turn("Reply with one word again.", cached=True),
+ ],
+ )
+
+
+def _assert_flagged_model_keeps_cache(
+ client: PassthroughClient, resources: ResourceManager, params: LiteLLMParamsBody
+) -> None:
+ model = _register_deployment(client, resources, params)
+ key = resources.key(models=[model])
+ system_turn = _cacheable_system_turn(unique_marker())
+
+ primed = _prime_prompt_cache(client, key, model, system_turn)
+
+ second = unwrap(_post_chat(client, key, _reminder_turn_body(model, system_turn, primed)))
+ read_tokens = _cache_read_tokens(second.usage)
+
+ assert _response_role(second) == "assistant", f"{model}: unexpected role {_response_role(second)!r}"
+ assert _response_text(second).strip(), f"{model}: reminder turn returned no completion text"
+ assert read_tokens >= primed.full_prefix_tokens, (
+ f"{model}: turn with a mid-conversation system reminder read {read_tokens} "
+ f"cached tokens, expected at least the {primed.full_prefix_tokens} cached on "
+ f"turn one ({primed.prefix_read_tokens} system prefix + "
+ f"{primed.first_turn_creation_tokens} first user turn); the reminder was "
+ f"hoisted into the top-level system field, which mutates the cached prefix "
+ f"and re-bills the conversation at cache-write pricing"
+ )
+
+
+def _assert_unflagged_model_converts_and_succeeds(
+ client: PassthroughClient, resources: ResourceManager, params: LiteLLMParamsBody
+) -> None:
+ model = _register_deployment(client, resources, params)
+ key = resources.key(models=[model])
+ system_turn = _cacheable_system_turn(unique_marker())
+
+ primed = _prime_prompt_cache(client, key, model, system_turn)
+
+ second = unwrap(_post_chat(client, key, _reminder_turn_body(model, system_turn, primed)))
+ read_tokens = _cache_read_tokens(second.usage)
+
+ assert _response_role(second) == "assistant", f"{model}: unexpected role {_response_role(second)!r}"
+ assert _response_text(second).strip(), (
+ f"{model}: conversation with a mid-conversation system reminder returned "
+ f"no text; the reminder was forwarded in place to a model that rejects "
+ f"role 'system' inside messages instead of being converted to a user turn"
+ )
+ assert read_tokens >= primed.full_prefix_tokens, (
+ f"{model}: reminder turn read {read_tokens} cached tokens, expected at least "
+ f"the {primed.full_prefix_tokens} cached on turn one "
+ f"({primed.prefix_read_tokens} system prefix + "
+ f"{primed.first_turn_creation_tokens} first user turn); the reminder was "
+ f"hoisted into the top-level system field instead of being converted to a "
+ f"user turn in place, mutating the cached prefix and re-billing the "
+ f"conversation at cache-write pricing"
+ )
+
+
+class TestAnthropicChatMidConversationSystem:
+ FLAGGED_MODEL = "anthropic/claude-opus-4-8"
+ UNFLAGGED_MODEL = "anthropic/claude-haiku-4-5-20251001"
+
+ @pytest.mark.covers(
+ "llm.chat_completions.anthropic.mid_conversation_system.nonstream.cache_hit",
+ exercised_on=[],
+ )
+ def test_flagged_model_keeps_prompt_cache_across_system_reminder(
+ self, client: PassthroughClient, resources: ResourceManager
+ ) -> None:
+ _assert_flagged_model_keeps_cache(client, resources, _anthropic_params(self.FLAGGED_MODEL))
+
+ @pytest.mark.covers(
+ "llm.chat_completions.anthropic.mid_conversation_system.nonstream.works",
+ exercised_on=[],
+ )
+ def test_unflagged_model_converts_system_reminder_and_succeeds(
+ self, client: PassthroughClient, resources: ResourceManager
+ ) -> None:
+ _assert_unflagged_model_converts_and_succeeds(client, resources, _anthropic_params(self.UNFLAGGED_MODEL))
+
+
+class TestBedrockInvokeChatMidConversationSystem:
+ FLAGGED_MODEL = "bedrock/invoke/us.anthropic.claude-sonnet-5"
+ UNFLAGGED_MODEL = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+ AWS_REGION = "us-east-1"
+
+ @pytest.mark.covers(
+ "llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.cache_hit",
+ exercised_on=[],
+ )
+ def test_flagged_model_keeps_prompt_cache_across_system_reminder(
+ self, client: PassthroughClient, resources: ResourceManager
+ ) -> None:
+ _assert_flagged_model_keeps_cache(client, resources, _invoke_params(self.FLAGGED_MODEL, self.AWS_REGION))
+
+ @pytest.mark.covers(
+ "llm.chat_completions.bedrock_invoke.mid_conversation_system.nonstream.works",
+ exercised_on=[],
+ )
+ def test_unflagged_model_converts_system_reminder_and_succeeds(
+ self, client: PassthroughClient, resources: ResourceManager
+ ) -> None:
+ _assert_unflagged_model_converts_and_succeeds(
+ client, resources, _invoke_params(self.UNFLAGGED_MODEL, self.AWS_REGION)
+ )
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index 4322662cfcb..26124ac24de 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -3847,3 +3847,51 @@ def test_convert_to_anthropic_tool_invoke_keeps_paired_server_tool_use():
},
server_result,
]
+
+
+def test_anthropic_messages_pt_keeps_system_role_after_user_turn():
+ """Models flagged supports_mid_conversation_system accept role=system inside
+ messages; the converter must emit it as a system message with its text
+ blocks and cache_control intact instead of rejecting the role."""
+ messages = [
+ {"role": "user", "content": "First question"},
+ {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}],
+ },
+ {"role": "assistant", "content": "Yes"},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ result = anthropic_messages_pt(messages=messages, model="claude-opus-4-8", llm_provider="anthropic")
+
+ assert [m["role"] for m in result] == ["user", "system", "assistant", "user"]
+ assert result[1] == {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer in one word.", "cache_control": {"type": "ephemeral"}}],
+ }
+
+
+def test_anthropic_messages_pt_system_string_content_becomes_text_block():
+ messages = [
+ {"role": "user", "content": "First question"},
+ {"role": "system", "content": "Answer in one word."},
+ ]
+
+ result = anthropic_messages_pt(messages=messages, model="claude-opus-4-8", llm_provider="anthropic")
+
+ assert result[1] == {"role": "system", "content": [{"type": "text", "text": "Answer in one word."}]}
+
+
+def test_anthropic_messages_pt_drops_a_system_message_with_no_text():
+ """Anthropic rejects empty text blocks, so a text-less system message must
+ vanish rather than reach the wire as an empty system turn."""
+ messages = [
+ {"role": "user", "content": "First question"},
+ {"role": "system", "content": ""},
+ {"role": "assistant", "content": "Yes"},
+ ]
+
+ result = anthropic_messages_pt(messages=messages, model="claude-opus-4-8", llm_provider="anthropic")
+
+ assert [m["role"] for m in result] == ["user", "assistant"]
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py
new file mode 100644
index 00000000000..d1e23a17747
--- /dev/null
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_mid_conversation_system.py
@@ -0,0 +1,392 @@
+"""Placement policy for mid-conversation ``role: "system"`` messages on the chat path.
+
+The provider-facing behaviour is covered through ``transform_request`` in the
+Anthropic, Vertex, Azure AI and Bedrock Invoke transformation tests; these pin
+the pure placement rules on the OpenAI-format message list.
+"""
+
+import pytest
+
+import litellm
+from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+ CONVERTED_SYSTEM_NOTE,
+ place_mid_conversation_system,
+ split_leading_system_run,
+)
+
+
+def _roles(messages: object) -> list[str]:
+ return [m["role"] if isinstance(m, dict) else m.role for m in messages]
+
+
+def _texts(message: dict) -> list[str]:
+ return [block["text"] for block in message["content"]]
+
+
+SENDS_NOTHING = pytest.mark.parametrize(
+ "empty_content",
+ [[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]],
+ ids=["empty-list", "none", "unsupported-part-only"],
+)
+
+
+def test_split_leading_system_run_keeps_later_system_messages_in_the_conversation():
+ messages = [
+ {"role": "system", "content": "one"},
+ {"role": "system", "content": "two"},
+ {"role": "user", "content": "q"},
+ {"role": "system", "content": "reminder"},
+ ]
+
+ leading, later = split_leading_system_run(messages)
+
+ assert [m["content"] for m in leading] == ["one", "two"]
+ assert _roles(later) == ["user", "system"]
+
+
+def test_flagged_placement_moves_a_system_run_after_the_user_turn_that_follows_it():
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": "q2"},
+ {"role": "assistant", "content": "a2"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=True)
+
+ assert _roles(placed) == ["user", "assistant", "user", "system", "assistant"]
+
+
+def test_flagged_placement_pushes_a_system_between_two_user_turns_after_both():
+ """Two user turns collapse into one on the wire, and a system message must
+ be followed by an assistant turn or nothing."""
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": "q2"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=True)
+
+ assert _roles(placed) == ["user", "user", "system"]
+
+
+def test_flagged_placement_keeps_a_system_after_tool_results():
+ messages = [
+ {"role": "user", "content": "q1"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
+ },
+ {"role": "tool", "tool_call_id": "c1", "content": "r"},
+ {"role": "system", "content": "reminder"},
+ {"role": "assistant", "content": "a2"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=True)
+
+ assert _roles(placed) == ["user", "assistant", "tool", "system", "assistant"]
+
+
+def test_flagged_placement_drops_a_system_message_with_no_text():
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": ""},
+ {"role": "assistant", "content": "a1"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=True)
+
+ assert _roles(placed) == ["user", "assistant"]
+
+
+def test_placement_reads_roles_off_pydantic_messages_in_the_history():
+ """Callers routinely append the previous ``litellm.Message`` object straight
+ into the history; placement must read its role without assuming a dict and
+ hand the object through untouched."""
+ assistant = litellm.Message(role="assistant", content="a1")
+ messages = [
+ {"role": "user", "content": "q1"},
+ assistant,
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": "q2"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=True)
+
+ assert _roles(placed) == ["user", "assistant", "user", "system"]
+ assert placed[1] is assistant
+
+
+def test_unflagged_conversion_keeps_the_client_order_when_no_tool_result_follows():
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": "q2"},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=False)
+
+ assert _roles(placed) == ["user", "assistant", "user", "user"]
+ assert _texts(placed[2]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
+
+
+@pytest.mark.parametrize(
+ "cache_control, expected",
+ [
+ ({"type": "ephemeral", "ttl": "1h"}, {"type": "ephemeral", "ttl": "1h"}),
+ ({"type": "ephemeral", "ttl": "5m"}, {"type": "ephemeral", "ttl": "5m"}),
+ ({"type": "ephemeral", "ttl": "2h"}, {"type": "ephemeral"}),
+ ],
+)
+def test_unflagged_conversion_rebuilds_cache_control_on_the_converted_block(cache_control, expected):
+ """Only the shapes Anthropic accepts survive: ephemeral with a 5m or 1h ttl, or no ttl."""
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder", "cache_control": cache_control},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=False)
+
+ assert placed[1]["content"][1] == {"type": "text", "text": "reminder", "cache_control": expected}
+
+
+def test_unflagged_conversion_drops_a_cache_control_that_is_not_ephemeral():
+ messages = [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder", "cache_control": {"type": "persistent"}},
+ ]
+
+ placed = place_mid_conversation_system(messages, supports_mid_conversation_system=False)
+
+ assert placed[1]["content"][1] == {"type": "text", "text": "reminder"}
+
+
+def test_placement_is_a_no_op_without_later_system_messages():
+ messages = [{"role": "user", "content": "q1"}, {"role": "assistant", "content": "a1"}]
+
+ assert place_mid_conversation_system(messages, supports_mid_conversation_system=False) == tuple(messages)
+ assert place_mid_conversation_system(messages, supports_mid_conversation_system=True) == tuple(messages)
+
+
+def test_flagged_placement_converts_a_run_followed_by_an_assistant_turn_in_place():
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "system", "content": "reminder"},
+ {"role": "assistant", "content": "a2"},
+ {"role": "user", "content": "q2"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "assistant", "user", "assistant", "user"]
+ assert _texts(placed[2]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
+
+
+def test_flagged_placement_of_an_earlier_run_does_not_move_when_later_turns_are_appended():
+ turn_n = [
+ {"role": "user", "content": "q1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "system", "content": "reminder"},
+ ]
+ turn_n_plus_one = [
+ *turn_n,
+ {"role": "assistant", "content": "a2"},
+ {"role": "user", "content": "q2"},
+ ]
+
+ placed_n = place_mid_conversation_system(turn_n, supports_mid_conversation_system=True)
+ placed_n_plus_one = place_mid_conversation_system(turn_n_plus_one, supports_mid_conversation_system=True)
+
+ assert placed_n_plus_one[: len(placed_n)] == placed_n
+ assert _roles(placed_n_plus_one) == ["user", "assistant", "user", "assistant", "user"]
+
+
+@SENDS_NOTHING
+def test_flagged_placement_converts_a_run_whose_preceding_user_turn_sends_nothing(empty_content):
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": empty_content},
+ {"role": "system", "content": "reminder"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "user", "content": "q2"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "user", "assistant", "user"]
+ assert _texts(placed[1]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
+
+
+@SENDS_NOTHING
+def test_flagged_placement_converts_a_run_whose_following_user_turn_sends_nothing(empty_content):
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "assistant", "content": "a1"},
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": empty_content},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "assistant", "user", "user"]
+ assert _texts(placed[2]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
+
+
+def test_flagged_placement_keeps_a_system_behind_a_user_turn_merged_with_an_empty_one():
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "user", "content": []},
+ {"role": "system", "content": "reminder"},
+ {"role": "assistant", "content": "a1"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "user", "system", "assistant"]
+
+
+EMPTY_ASSISTANT = pytest.mark.parametrize(
+ "empty_assistant",
+ [
+ {"role": "assistant", "content": None},
+ {"role": "assistant", "content": []},
+ {"role": "assistant", "content": [{"type": "thinking", "thinking": "unsigned"}]},
+ {
+ "role": "assistant",
+ "content": [{"type": "thinking", "thinking": "bridged", "signature": encrypted_reasoning_signature("abc")}],
+ },
+ {
+ "role": "assistant",
+ "content": None,
+ "thinking_blocks": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("abc")}],
+ },
+ {
+ "role": "assistant",
+ "content": [{"type": "thinking", "thinking": "unsigned"}],
+ "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+ {
+ "role": "assistant",
+ "content": [{"type": "redacted_thinking", "data": "x"}],
+ "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}],
+ },
+ litellm.Message(role="assistant", content=None),
+ ],
+ ids=[
+ "none",
+ "empty-list",
+ "unsigned-thinking-part",
+ "encrypted-thinking-part",
+ "encrypted-redacted-thinking-block",
+ "unsigned-inline-part-hides-signed-block",
+ "inline-redacted-part-hides-redacted-block",
+ "pydantic-none",
+ ],
+)
+
+
+@EMPTY_ASSISTANT
+def test_flagged_placement_converts_a_run_when_the_assistant_turn_after_its_anchor_sends_nothing(empty_assistant):
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder"},
+ empty_assistant,
+ {"role": "user", "content": "q2"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "user", "assistant", "user"]
+ assert _texts(placed[1]) == [CONVERTED_SYSTEM_NOTE, "reminder"]
+ assert placed[2] is empty_assistant
+
+
+@EMPTY_ASSISTANT
+def test_flagged_placement_keeps_a_system_whose_empty_assistant_follower_ends_the_array(empty_assistant):
+ placed = place_mid_conversation_system(
+ [{"role": "user", "content": "q1"}, {"role": "system", "content": "reminder"}, empty_assistant],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "system", "assistant"]
+
+
+@pytest.mark.parametrize(
+ "assistant_turn",
+ [
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
+ },
+ {
+ "role": "assistant",
+ "content": None,
+ "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+ {"role": "assistant", "content": None, "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}]},
+ {"role": "assistant", "content": [{"type": "thinking", "thinking": "hm", "signature": "s"}]},
+ {"role": "assistant", "content": None, "function_call": {"name": "f", "arguments": "{}"}},
+ litellm.Message(
+ role="assistant",
+ content="",
+ tool_calls=[{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
+ ),
+ ],
+ ids=[
+ "tool-calls",
+ "signed-thinking-block",
+ "redacted-thinking-block",
+ "signed-thinking-part",
+ "function-call",
+ "pydantic-tool-calls",
+ ],
+)
+def test_flagged_placement_keeps_a_system_before_an_assistant_turn_that_renders_without_text(assistant_turn):
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder"},
+ assistant_turn,
+ {"role": "user", "content": "q2"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "system", "assistant", "user"]
+
+
+@pytest.mark.parametrize(
+ "padded_assistant",
+ [
+ {"role": "assistant", "content": ""},
+ {"role": "assistant", "content": " "},
+ {"role": "assistant", "content": [{"type": "text", "text": ""}]},
+ litellm.Message(role="assistant", content=""),
+ ],
+ ids=["empty-string", "whitespace-string", "empty-text-part", "pydantic-empty-string"],
+)
+def test_flagged_placement_keeps_a_system_before_an_assistant_turn_whose_empty_text_the_converter_pads(
+ padded_assistant,
+):
+ placed = place_mid_conversation_system(
+ [
+ {"role": "user", "content": "q1"},
+ {"role": "system", "content": "reminder"},
+ padded_assistant,
+ {"role": "user", "content": "q2"},
+ ],
+ supports_mid_conversation_system=True,
+ )
+
+ assert _roles(placed) == ["user", "system", "assistant", "user"]
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index bb8098e6e23..3afa31cc801 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -1,5 +1,6 @@
import asyncio
import contextlib
+import copy
import datetime
import json
import logging
@@ -8781,3 +8782,41 @@ def test_extract_response_obj_and_hidden_params_reads_binary_content_hidden_para
assert hidden_params == {"headers": {"x-request-id": "req_tts"}}
assert response_obj["object"] == "binary"
+
+
+def _preserved_thinking_client_turns() -> tuple[list[dict], list[dict]]:
+ turn_n = [{"role": "user", "content": "First question"}]
+ reply = {
+ "role": "assistant",
+ "content": "First answer",
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": "sig-1"}],
+ }
+ return turn_n, [*turn_n, reply, {"role": "user", "content": "Second question"}]
+
+
+@pytest.mark.asyncio
+async def test_prompt_management_with_unchanged_variables_replays_a_byte_identical_prefix(logging_obj, tmp_path):
+ """A prompt template rendered with the same variables on every turn must prepend the
+ same messages, or the signed thinking blocks in the history lose their binding."""
+ from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager
+
+ (tmp_path / "greeting.prompt").write_text(
+ "---\nmodel: claude-fable-5-1\n---\nSystem: You are a {{persona}}. Answer in one sentence.\n"
+ )
+ manager = DotpromptManager(prompt_directory=str(tmp_path))
+ compiled = [
+ await logging_obj.async_get_chat_completion_prompt(
+ model="claude-fable-5-1",
+ messages=copy.deepcopy(turn),
+ non_default_params={},
+ prompt_variables={"persona": "pirate"},
+ prompt_id="greeting",
+ prompt_management_logger=manager,
+ )
+ for turn in _preserved_thinking_client_turns()
+ ]
+ (_, messages_n, _), (_, messages_n_plus_one, _) = compiled
+
+ assert json.dumps(messages_n_plus_one[: len(messages_n)], sort_keys=True) == json.dumps(messages_n, sort_keys=True)
+ assert messages_n[0] == {"role": "system", "content": "You are a pirate. Answer in one sentence."}
+ assert len(messages_n_plus_one) == len(messages_n) + 2
diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
index 7167a67d80d..729f46ec57f 100644
--- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
+++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py
@@ -1,4 +1,4 @@
-
+import copy
import json
from typing import Final
from unittest.mock import MagicMock, patch
@@ -17,10 +17,18 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
)
+from litellm.litellm_core_utils.prompt_templates.common_utils import encrypted_reasoning_signature
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
+from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
+from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
+ AmazonAnthropicClaudeConfig,
+)
+from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import (
+ VertexAIAnthropicConfig,
+)
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
from litellm.types.utils import ServerToolUse, Usage
@@ -37,13 +45,9 @@ def test_response_format_transformation_unit_test():
"additionalProperties": False,
}
- result = config._create_json_tool_call_for_response_format(
- json_schema=response_format_json_schema
- )
+ result = config._create_json_tool_call_for_response_format(json_schema=response_format_json_schema)
- assert result["input_schema"]["properties"] == {
- "agent_doing": {"title": "Agent Doing", "type": "string"}
- }
+ assert result["input_schema"]["properties"] == {"agent_doing": {"title": "Agent Doing", "type": "string"}}
print(result)
@@ -128,7 +132,9 @@ def test_calculate_usage_prefers_served_speed_from_response_usage():
assert no_response_speed.speed == "fast"
-@pytest.mark.parametrize("input_update, expected_fresh", [({}, 1000), ({"input_tokens": 0}, 0), ({"input_tokens": 2000}, 2000)])
+@pytest.mark.parametrize(
+ "input_update, expected_fresh", [({}, 1000), ({"input_tokens": 0}, 0), ({"input_tokens": 2000}, 2000)]
+)
def test_streaming_iterator_persists_cumulative_usage_across_partial_chunks(input_update, expected_fresh):
"""
Omitted input/cache/pricing fields retain their last cumulative values;
@@ -138,11 +144,17 @@ def test_streaming_iterator_persists_cumulative_usage_across_partial_chunks(inpu
iterator = ModelResponseIterator(None, sync_stream=True, speed="fast")
- start_usage = iterator._handle_usage({
- "input_tokens": 1000, "output_tokens": 1, "speed": "standard", "inference_geo": "us",
- "cache_creation_input_tokens": 3000, "cache_read_input_tokens": 2000,
- "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3000},
- })
+ start_usage = iterator._handle_usage(
+ {
+ "input_tokens": 1000,
+ "output_tokens": 1,
+ "speed": "standard",
+ "inference_geo": "us",
+ "cache_creation_input_tokens": 3000,
+ "cache_read_input_tokens": 2000,
+ "cache_creation": {"ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 3000},
+ }
+ )
delta_usage = iterator._handle_usage({"output_tokens": 5, **input_update})
assert start_usage.speed == "standard"
@@ -564,9 +576,7 @@ def test_extract_response_content_with_citations():
},
}
- _, citations, _, _, _, _, _, _ = config.extract_response_content(
- completion_response
- )
+ _, citations, _, _, _, _, _, _ = config.extract_response_content(completion_response)
assert citations == [
[
{
@@ -639,12 +649,8 @@ def test_web_search_tool_transformation():
assert anthropic_web_search_tool["user_location"]["city"] == "San Francisco"
-@pytest.mark.parametrize(
- "search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)]
-)
-def test_web_search_tool_transformation_with_search_context_size(
- search_context_size, expected_max_uses
-):
+@pytest.mark.parametrize("search_context_size, expected_max_uses", [("low", 1), ("medium", 5), ("high", 10)])
+def test_web_search_tool_transformation_with_search_context_size(search_context_size, expected_max_uses):
from litellm.types.llms.openai import OpenAIWebSearchOptions
config = AnthropicConfig()
@@ -819,10 +825,7 @@ def test_web_search_tool_result_in_provider_specific_fields():
assert "web_search_results" in provider_fields
assert len(provider_fields["web_search_results"]) == 1
assert provider_fields["web_search_results"][0]["type"] == "web_search_tool_result"
- assert (
- provider_fields["web_search_results"][0]["tool_use_id"]
- == "srvtoolu_provider_test"
- )
+ assert provider_fields["web_search_results"][0]["tool_use_id"] == "srvtoolu_provider_test"
def test_multiple_web_search_tool_results():
@@ -1046,10 +1049,7 @@ def test_transform_response_with_prefix_prompt():
)
assert result is not None
- assert (
- result.choices[0].message.content
- == "You are a helpful assistant. The grass is green."
- )
+ assert result.choices[0].message.content == "You are a helpful assistant. The grass is green."
def test_get_supported_params_thinking():
@@ -1164,18 +1164,12 @@ def test_anthropic_beta_header_merging_with_output_format():
}
}
- result_headers = config.update_headers_with_optional_anthropic_beta(
- headers, optional_params
- )
+ result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params)
# Both beta headers should be present
beta_value = result_headers["anthropic-beta"]
- assert (
- "context-1m-2025-08-07" in beta_value
- ), f"User's context-1m beta header missing from: {beta_value}"
- assert (
- "structured-outputs-2025-11-13" in beta_value
- ), f"Structured output beta header missing from: {beta_value}"
+ assert "context-1m-2025-08-07" in beta_value, f"User's context-1m beta header missing from: {beta_value}"
+ assert "structured-outputs-2025-11-13" in beta_value, f"Structured output beta header missing from: {beta_value}"
def test_anthropic_beta_header_merging_with_multiple_features():
@@ -1197,9 +1191,7 @@ def test_anthropic_beta_header_merging_with_multiple_features():
"tools": [{"type": "web_fetch_20250910", "name": "web_fetch"}],
}
- result_headers = config.update_headers_with_optional_anthropic_beta(
- headers, optional_params
- )
+ result_headers = config.update_headers_with_optional_anthropic_beta(headers, optional_params)
beta_value = result_headers["anthropic-beta"]
@@ -1242,9 +1234,7 @@ def test_anthropic_structured_output_beta_header():
"strict": True,
"schema": {
"description": 'Progress report for the thinking process\n\nThis model represents a snapshot of the agent\'s current progress during\nthe thinking process, providing a brief description of the current activity.\n\nAttributes:\n agent_doing: Brief description of what the agent is currently doing.\n Should be kept under 10 words. Example: "Learning about home automation"',
- "properties": {
- "agent_doing": {"title": "Agent Doing", "type": "string"}
- },
+ "properties": {"agent_doing": {"title": "Agent Doing", "type": "string"}},
"required": ["agent_doing"],
"title": "ThinkingStep",
"type": "object",
@@ -1258,10 +1248,7 @@ def test_anthropic_structured_output_beta_header():
assert response is not None
print(f"response: {response}")
print(f"raw_request_headers: {response['raw_request_headers']}")
- assert (
- "structured-outputs-2025-11-13"
- in response["raw_request_headers"]["anthropic-beta"]
- )
+ assert "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"]
@pytest.mark.parametrize(
@@ -1397,9 +1384,7 @@ def test_tool_search_regex_detection():
config = AnthropicModelInfo()
# Test with tool search regex tool
- tools = [
- {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}
- ]
+ tools = [{"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}]
assert config.is_tool_search_used(tools) is True
# Test without tool search
@@ -1414,9 +1399,7 @@ def test_tool_search_bm25_detection():
config = AnthropicModelInfo()
# Test with tool search BM25 tool
- tools = [
- {"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}
- ]
+ tools = [{"type": "tool_search_tool_bm25_20251119", "name": "tool_search_tool_bm25"}]
assert config.is_tool_search_used(tools) is True
@@ -1608,9 +1591,7 @@ def test_tool_search_complete_response_parsing():
"tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ",
"content": {
"type": "tool_search_tool_search_result",
- "tool_references": [
- {"type": "tool_reference", "tool_name": "get_weather"}
- ],
+ "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}],
},
},
{"type": "text", "text": "Great! I found a weather tool."},
@@ -1661,9 +1642,7 @@ def test_tool_search_complete_response_parsing():
assert usage.server_tool_use is not None
assert usage.server_tool_use.web_search_requests == 0
- assert (
- usage.server_tool_use.tool_search_requests == 1
- ) # Counted from server_tool_use blocks
+ assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks
def test_allowed_callers_field_preservation():
@@ -1715,9 +1694,7 @@ def test_programmatic_tool_calling_beta_header():
assert is_programmatic is True
# Test header generation
- headers = model_info.get_anthropic_headers(
- api_key="test-key", programmatic_tool_calling_used=True
- )
+ headers = model_info.get_anthropic_headers(api_key="test-key", programmatic_tool_calling_used=True)
assert "anthropic-beta" in headers
assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"]
@@ -1861,9 +1838,7 @@ def test_input_examples_beta_header():
assert is_examples_used is True
# Test header generation
- headers = model_info.get_anthropic_headers(
- api_key="test-key", input_examples_used=True
- )
+ headers = model_info.get_anthropic_headers(api_key="test-key", input_examples_used=True)
assert "anthropic-beta" in headers
assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"]
@@ -1949,10 +1924,7 @@ def test_input_examples_empty_list_not_added():
transformed_tool, _ = config._map_tool_helper(tool)
assert transformed_tool is not None
# Empty list should not be added
- assert (
- "input_examples" not in transformed_tool
- or len(transformed_tool.get("input_examples", [])) == 0
- )
+ assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0
# ============ Effort Parameter Tests ============
@@ -2012,9 +1984,7 @@ def test_effort_beta_header_injection():
effort_used = model_info.is_effort_used(optional_params=optional_params, custom_llm_provider="anthropic")
assert effort_used is True
- headers = model_info.get_anthropic_headers(
- api_key="test-key", effort_used=effort_used
- )
+ headers = model_info.get_anthropic_headers(api_key="test-key", effort_used=effort_used)
assert "anthropic-beta" in headers
assert "effort-2025-11-24" in headers["anthropic-beta"]
@@ -2040,9 +2010,7 @@ def test_effort_validation():
optional_params = {"output_config": {"effort": "invalid"}}
- with pytest.raises(
- litellm.exceptions.BadRequestError, match="Invalid effort value"
- ):
+ with pytest.raises(litellm.exceptions.BadRequestError, match="Invalid effort value"):
config.transform_request(
model="claude-opus-4-5-20251101",
messages=messages,
@@ -2278,16 +2246,8 @@ def test_anthropic_model_supports_speed_param_rejects_non_anthropic_providers(
):
"""Fast mode is direct-Anthropic-only. Vertex/Azure/Bedrock strip their prefix
before the shared transform runs, so the bare Opus id must still be rejected."""
- assert (
- AnthropicConfig._model_supports_speed_param(
- "claude-opus-4-8", custom_llm_provider
- )
- is False
- )
- assert (
- AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic")
- is True
- )
+ assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", custom_llm_provider) is False
+ assert AnthropicConfig._model_supports_speed_param("claude-opus-4-8", "anthropic") is True
def test_vertex_anthropic_drops_speed_for_opus_with_drop_params(monkeypatch):
@@ -2571,9 +2531,7 @@ def test_supports_effort_level_handles_provider_prefixes(model, level, expected)
("claude-opus-4-5-20251101", None, False),
],
)
-def test_validate_effort_for_model_centralises_per_model_gating(
- model, effort, expect_error
-):
+def test_validate_effort_for_model_centralises_per_model_gating(model, effort, expect_error):
err = AnthropicConfig._validate_effort_for_model(model, effort, "anthropic")
if expect_error:
assert err is not None
@@ -2622,11 +2580,7 @@ def test_transform_request_injects_dummy_tool_without_tools_param():
litellm.modify_params = prev_modify_params
assert "tools" in result
- names = [
- t.get("name")
- for t in result["tools"]
- if isinstance(t, dict) and t.get("name") is not None
- ]
+ names = [t.get("name") for t in result["tools"] if isinstance(t, dict) and t.get("name") is not None]
assert "dummy_tool" in names
@@ -2692,13 +2646,9 @@ def test_calculate_usage_completion_tokens_details_with_reasoning():
"output_tokens": 500,
}
# Simulating reasoning content that would count as ~50 tokens
- reasoning_content = (
- "Let me think about this step by step. " * 10
- ) # Roughly 50 tokens
+ reasoning_content = "Let me think about this step by step. " * 10 # Roughly 50 tokens
- usage = config.calculate_usage(
- usage_object=usage_object, reasoning_content=reasoning_content
- )
+ usage = config.calculate_usage(usage_object=usage_object, reasoning_content=reasoning_content)
# completion_tokens_details should be populated with both reasoning and text tokens
assert usage.completion_tokens_details is not None
@@ -2749,9 +2699,7 @@ def test_reasoning_effort_maps_to_adaptive_thinking_for_claude_4_6_models():
# reasoning_effort should not be in the result (it's transformed to thinking)
assert "reasoning_effort" not in result
# Should set output_config with the mapped effort value
- assert (
- "output_config" in result
- ), f"output_config missing for {model} with effort={effort}"
+ assert "output_config" in result, f"output_config missing for {model} with effort={effort}"
assert result["output_config"]["effort"] == effort_map[effort]
@@ -2852,9 +2800,7 @@ def test_raw_adaptive_thinking_untouched_for_46_plus_model():
("gpt-4o", False),
],
)
-def test_is_adaptive_thinking_model_is_sourced_from_cost_map(
- local_model_cost_map, model, expected
-):
+def test_is_adaptive_thinking_model_is_sourced_from_cost_map(local_model_cost_map, model, expected):
"""Adaptive thinking resolves from the cost map first (an explicit
supports_adaptive_thinking entry, or the anthropic-claude fallback rule for unmapped
future Claudes), then from a date-safe opus/sonnet/haiku >= 4.6 name version as a
@@ -2970,9 +2916,7 @@ def test_reasoning_effort_sets_output_config_for_46_models():
drop_params=False,
)
- assert (
- "output_config" in result
- ), f"output_config missing for {model} with effort={effort}"
+ assert "output_config" in result, f"output_config missing for {model} with effort={effort}"
assert result["output_config"]["effort"] == effort
@@ -3011,9 +2955,7 @@ def test_reasoning_effort_does_not_set_output_config_for_older_models():
drop_params=False,
)
- assert (
- "output_config" not in result
- ), f"output_config should not be set for {model}"
+ assert "output_config" not in result, f"output_config should not be set for {model}"
@pytest.mark.parametrize(
@@ -3053,14 +2995,10 @@ def test_reasoning_effort_accepts_dict_shape_for_adaptive_model(reasoning_effort
)
# thinking must be set (adaptive for 4.6+)
- assert (
- "thinking" in result
- ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}"
+ assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}"
assert result["thinking"]["type"] == "adaptive"
# output_config must carry the mapped effort
- assert (
- "output_config" in result
- ), f"output_config missing for reasoning_effort={reasoning_effort_value!r}"
+ assert "output_config" in result, f"output_config missing for reasoning_effort={reasoning_effort_value!r}"
assert result["output_config"]["effort"] == "low"
@@ -3089,16 +3027,13 @@ def test_reasoning_effort_accepts_dict_shape_for_non_adaptive_model(
drop_params=False,
)
- assert (
- "thinking" in result
- ), f"thinking missing for reasoning_effort={reasoning_effort_value!r}"
+ assert "thinking" in result, f"thinking missing for reasoning_effort={reasoning_effort_value!r}"
assert result["thinking"]["type"] == "enabled"
assert "budget_tokens" in result["thinking"]
assert result["thinking"]["budget_tokens"] > 0
# Older models must not get adaptive-thinking output_config
assert "output_config" not in result, (
- f"output_config should not be set for non-adaptive model "
- f"(reasoning_effort={reasoning_effort_value!r})"
+ f"output_config should not be set for non-adaptive model (reasoning_effort={reasoning_effort_value!r})"
)
@@ -3149,12 +3084,8 @@ def test_reasoning_effort_unparseable_dict_is_dropped(bad_value):
model="claude-sonnet-4-6-20260219",
drop_params=False,
)
- assert (
- "thinking" not in result
- ), f"thinking should not be set for bad value {bad_value!r}"
- assert (
- "output_config" not in result
- ), f"output_config should not be set for bad value {bad_value!r}"
+ assert "thinking" not in result, f"thinking should not be set for bad value {bad_value!r}"
+ assert "output_config" not in result, f"output_config should not be set for bad value {bad_value!r}"
@pytest.mark.parametrize(
@@ -3285,9 +3216,7 @@ def test_reasoning_effort_garbage_raises_bad_request(effort):
("max", DEFAULT_REASONING_EFFORT_MAX_THINKING_BUDGET),
],
)
-def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model(
- effort, expected_budget
-):
+def test_reasoning_effort_xhigh_max_maps_to_budget_on_budget_model(effort, expected_budget):
"""``xhigh`` / ``max`` extend the budget_tokens progression on budget-mode models."""
config = AnthropicConfig()
@@ -3434,17 +3363,11 @@ def test_code_execution_tool_results_extraction():
# Verify first tool call
assert transformed_response.choices[0].message.tool_calls[0].id == "srvtoolu_01ABC"
- assert (
- transformed_response.choices[0].message.tool_calls[0].function.name
- == "bash_code_execution"
- )
+ assert transformed_response.choices[0].message.tool_calls[0].function.name == "bash_code_execution"
# Verify second tool call
assert transformed_response.choices[0].message.tool_calls[1].id == "srvtoolu_01DEF"
- assert (
- transformed_response.choices[0].message.tool_calls[1].function.name
- == "text_editor_code_execution"
- )
+ assert transformed_response.choices[0].message.tool_calls[1].function.name == "text_editor_code_execution"
# Verify tool results are in provider_specific_fields
provider_fields = transformed_response.choices[0].message.provider_specific_fields
@@ -3467,10 +3390,7 @@ def test_code_execution_tool_results_extraction():
assert editor_result["content"]["is_file_update"] is False
# Verify text content is properly concatenated
- assert (
- "I'll calculate that for you."
- in transformed_response.choices[0].message.content
- )
+ assert "I'll calculate that for you." in transformed_response.choices[0].message.content
assert "Done!" in transformed_response.choices[0].message.content
@@ -3538,10 +3458,7 @@ def test_code_execution_tool_results_in_hidden_params():
assert "provider_specific_fields" in hidden
assert "tool_results" in hidden["provider_specific_fields"]
assert len(hidden["provider_specific_fields"]["tool_results"]) == 1
- assert (
- hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"]
- == "hello\n"
- )
+ assert hidden["provider_specific_fields"]["tool_results"][0]["content"]["stdout"] == "hello\n"
def test_tool_search_tool_result_not_in_tool_results():
@@ -3737,10 +3654,7 @@ def test_compaction_block_in_provider_specific_fields():
assert "compaction_blocks" in provider_fields
assert len(provider_fields["compaction_blocks"]) == 1
assert provider_fields["compaction_blocks"][0]["type"] == "compaction"
- assert (
- "Summary of the conversation"
- in provider_fields["compaction_blocks"][0]["content"]
- )
+ assert "Summary of the conversation" in provider_fields["compaction_blocks"][0]["content"]
def test_multiple_compaction_blocks():
@@ -3775,12 +3689,22 @@ def test_multiple_compaction_blocks():
assert compaction_blocks[1]["content"] == "Second summary..."
-@pytest.mark.parametrize("messages_api,gateway,native_endpoint", [
- (False, False, False), (True, False, False), (False, True, False), (True, True, False), (True, True, True),
-])
+@pytest.mark.parametrize(
+ "messages_api,gateway,native_endpoint",
+ [
+ (False, False, False),
+ (True, False, False),
+ (False, True, False),
+ (True, True, False),
+ (True, True, True),
+ ],
+)
async def test_native_compaction_wire_roundtrip(
- messages_api: bool, gateway: bool, native_endpoint: bool,
- monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter,
+ messages_api: bool,
+ gateway: bool,
+ native_endpoint: bool,
+ monkeypatch: pytest.MonkeyPatch,
+ respx_mock: respx.MockRouter,
) -> None:
monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True")
monkeypatch.setenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
@@ -3788,8 +3712,11 @@ async def test_native_compaction_wire_roundtrip(
monkeypatch.setattr(litellm, "use_chat_completions_url_for_anthropic_messages", False)
block: Final = {"type": "compaction", "content": "Exact summary", "signature": "opaque-signature"}
operation: Final = {"type": "summarize", "instructions": "Keep identifiers"}
- usage: Final = {"input_tokens": 0, "output_tokens": 0,
- "iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}]}
+ usage: Final = {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "iterations": [{"type": "compaction", "input_tokens": 103, "output_tokens": 165}],
+ }
chat_wire: Final = gateway and not native_endpoint
base: Final = "https://gateway.test/v1" if gateway else "https://api.anthropic.com/v1"
route: Final = respx_mock.post(f"{base}/{'chat/completions' if chat_wire else 'messages'}")
@@ -3798,27 +3725,51 @@ async def test_native_compaction_wire_roundtrip(
payload: Final = json.loads(request.content)
assert len(request.headers.get_list("anthropic-beta")) == 1
assert {value.strip() for value in request.headers["anthropic-beta"].split(",")} == {
- "compact-2026-09-04", "interleaved-thinking-2025-05-14",
+ "compact-2026-09-04",
+ "interleaved-thinking-2025-05-14",
}
if "compaction" in payload:
assert payload["compaction"] == operation
else:
assert payload["messages"][0] == {"role": "assistant", "content": [block]}
body: Final = (
- {"id": "chatcmpl_compact", "object": "chat.completion", "created": 1, "model": "claude-sonnet-5",
- "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "",
- "provider_specific_fields": {"compaction_blocks": [block]}}}],
- "usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268}}
- if chat_wire else
- {"id": "msg_compact", "type": "message", "role": "assistant", "model": "claude-sonnet-5",
- "content": [block], "stop_reason": "compaction", "usage": usage}
+ {
+ "id": "chatcmpl_compact",
+ "object": "chat.completion",
+ "created": 1,
+ "model": "claude-sonnet-5",
+ "choices": [
+ {
+ "index": 0,
+ "finish_reason": "stop",
+ "message": {
+ "role": "assistant",
+ "content": "",
+ "provider_specific_fields": {"compaction_blocks": [block]},
+ },
+ }
+ ],
+ "usage": {"prompt_tokens": 103, "completion_tokens": 165, "total_tokens": 268},
+ }
+ if chat_wire
+ else {
+ "id": "msg_compact",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-5",
+ "content": [block],
+ "stop_reason": "compaction",
+ "usage": usage,
+ }
)
return httpx.Response(200, json=body)
route.mock(side_effect=respond)
call: Final = litellm.anthropic.messages.acreate if messages_api else litellm.acompletion
params: Final = dict(
- model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5", api_key="test", max_tokens=512,
+ model=f"{'openai/' if gateway else ''}anthropic/claude-sonnet-5",
+ api_key="test",
+ max_tokens=512,
api_base=base if gateway else "https://api.anthropic.com",
extra_headers={"Anthropic-Beta": f"interleaved-thinking-2025-05-14{',compact-2026-09-04' if gateway else ''}"},
model_info={"supported_endpoints": ["/v1/messages"]} if native_endpoint else {},
@@ -3852,9 +3803,7 @@ def test_compaction_block_request_transformation():
{"role": "user", "content": "What is the weather in San Francisco?"},
{
"role": "assistant",
- "content": [
- {"type": "text", "text": "I don't have access to real-time data."}
- ],
+ "content": [{"type": "text", "text": "I don't have access to real-time data."}],
"provider_specific_fields": {
"compaction_blocks": [
{
@@ -3867,9 +3816,7 @@ def test_compaction_block_request_transformation():
{"role": "user", "content": "What about New York?"},
]
- result = anthropic_messages_pt(
- messages=messages, model="claude-opus-4-6", llm_provider="anthropic"
- )
+ result = anthropic_messages_pt(messages=messages, model="claude-opus-4-6", llm_provider="anthropic")
# Find the assistant message
assistant_message = None
@@ -3983,9 +3930,7 @@ def test_map_openai_context_management_to_anthropic():
"instructions": "Focus on preserving code snippets",
}
]
- result = config.map_openai_context_management_to_anthropic(
- openai_format_with_instructions
- )
+ result = config.map_openai_context_management_to_anthropic(openai_format_with_instructions)
assert result is not None
assert result["edits"][0]["trigger"]["value"] == 150000
@@ -4012,9 +3957,7 @@ def test_map_openai_params_with_context_management():
config = AnthropicConfig()
# Test with OpenAI list format
- non_default_params = {
- "context_management": [{"type": "compaction", "compact_threshold": 200000}]
- }
+ non_default_params = {"context_management": [{"type": "compaction", "compact_threshold": 200000}]}
optional_params = {}
result = config.map_openai_params(
@@ -4051,10 +3994,7 @@ def test_map_openai_params_with_context_management():
)
assert "context_management" in result
- assert (
- result["context_management"]
- == non_default_params_anthropic["context_management"]
- )
+ assert result["context_management"] == non_default_params_anthropic["context_management"]
def test_cache_control_in_supported_params():
@@ -4165,10 +4105,7 @@ def test_compaction_block_empty_list_not_added():
# Verify compaction_blocks is not in provider_specific_fields when there are none
provider_fields = result.choices[0].message.provider_specific_fields
if provider_fields:
- assert (
- "compaction_blocks" not in provider_fields
- or provider_fields.get("compaction_blocks") is None
- )
+ assert "compaction_blocks" not in provider_fields or provider_fields.get("compaction_blocks") is None
def test_fast_mode_beta_header():
@@ -4217,9 +4154,7 @@ def test_fast_mode_usage_calculation():
"output_tokens": 500,
}
- usage = config.calculate_usage(
- usage_object=usage_object, reasoning_content=None, speed="fast"
- )
+ usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None, speed="fast")
assert usage.prompt_tokens == 1000
assert usage.completion_tokens == 500
@@ -4240,9 +4175,7 @@ def test_fast_mode_cost_calculation():
base_completion = 0.025
with (
- patch(
- "litellm.llms.anthropic.cost_calculation.generic_cost_per_token"
- ) as mock_cost,
+ patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost,
patch("litellm.get_model_info") as mock_info,
):
mock_cost.return_value = (base_prompt, base_completion)
@@ -4282,9 +4215,7 @@ def test_fast_mode_with_inference_geo():
base_completion = 0.025
with (
- patch(
- "litellm.llms.anthropic.cost_calculation.generic_cost_per_token"
- ) as mock_cost,
+ patch("litellm.llms.anthropic.cost_calculation.generic_cost_per_token") as mock_cost,
patch("litellm.get_model_info") as mock_info,
):
mock_cost.return_value = (base_prompt, base_completion)
@@ -4475,9 +4406,7 @@ def test_map_tool_helper_enforces_object_type_when_missing():
"name": "search_code",
"description": "Search for code patterns",
"parameters": {
- "properties": {
- "query": {"type": "string", "description": "Search query"}
- },
+ "properties": {"query": {"type": "string", "description": "Search query"}},
"required": ["query"],
},
},
@@ -4490,9 +4419,9 @@ def test_map_tool_helper_enforces_object_type_when_missing():
assert "properties" in result["input_schema"]
assert "query" in result["input_schema"]["properties"]
# Original parameters dict must not be modified in place
- assert (
- tool["function"]["parameters"] == original_params
- ), "parameters dict was mutated; _map_tool_helper should not modify caller data"
+ assert tool["function"]["parameters"] == original_params, (
+ "parameters dict was mutated; _map_tool_helper should not modify caller data"
+ )
def test_map_tool_helper_enforces_object_type_when_wrong_type():
@@ -4518,13 +4447,13 @@ def test_map_tool_helper_enforces_object_type_when_wrong_type():
result, _ = config._map_tool_helper(tool)
assert result is not None
assert result["input_schema"]["type"] == "object"
- assert (
- result["input_schema"].get("properties") == {}
- ), "properties should be injected as {} when schema has non-object type and no properties key"
+ assert result["input_schema"].get("properties") == {}, (
+ "properties should be injected as {} when schema has non-object type and no properties key"
+ )
# Original parameters dict must not be modified in place
- assert (
- tool["function"]["parameters"] == original_params
- ), "parameters dict was mutated; _map_tool_helper should not modify caller data"
+ assert tool["function"]["parameters"] == original_params, (
+ "parameters dict was mutated; _map_tool_helper should not modify caller data"
+ )
def test_map_tool_helper_preserves_valid_object_schema():
@@ -4591,12 +4520,8 @@ def test_extract_response_content_thinking_block_null_thinking():
{"type": "text", "text": "Hello"},
]
}
- text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(
- completion_response_null
- )
- assert (
- thinking_blocks is not None
- ), "thinking blocks should not be None when thinking=null"
+ text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_null)
+ assert thinking_blocks is not None, "thinking blocks should not be None when thinking=null"
assert len(thinking_blocks) == 1
assert "Hello" in text
@@ -4607,12 +4532,8 @@ def test_extract_response_content_thinking_block_null_thinking():
{"type": "text", "text": "World"},
]
}
- text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(
- completion_response_missing
- )
- assert (
- thinking_blocks is not None
- ), "thinking blocks should not be None when thinking key is absent"
+ text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_missing)
+ assert thinking_blocks is not None, "thinking blocks should not be None when thinking key is absent"
assert len(thinking_blocks) == 1
assert "World" in text
@@ -4623,9 +4544,7 @@ def test_extract_response_content_thinking_block_null_thinking():
{"type": "text", "text": "Done"},
]
}
- text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(
- completion_response_text
- )
+ text, _, thinking_blocks, _, _, _, _, _ = config.extract_response_content(completion_response_text)
assert thinking_blocks is not None
assert len(thinking_blocks) == 1
assert thinking_blocks[0]["thinking"] == "Let me think..."
@@ -4684,12 +4603,8 @@ def test_advisor_beta_header_injected():
}
]
}
- result = config.update_headers_with_optional_anthropic_beta(
- headers, optional_params
- )
- assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get(
- "anthropic-beta", ""
- )
+ result = config.update_headers_with_optional_anthropic_beta(headers, optional_params)
+ assert ANTHROPIC_BETA_HEADER_VALUES.ADVISOR_TOOL_2026_03_01.value in result.get("anthropic-beta", "")
def test_advisor_beta_header_not_injected_without_tool():
@@ -4697,9 +4612,7 @@ def test_advisor_beta_header_not_injected_without_tool():
config = AnthropicConfig()
headers: dict = {}
optional_params: dict = {"tools": []}
- result = config.update_headers_with_optional_anthropic_beta(
- headers, optional_params
- )
+ result = config.update_headers_with_optional_anthropic_beta(headers, optional_params)
assert "advisor-tool-2026-03-01" not in result.get("anthropic-beta", "")
@@ -4726,9 +4639,7 @@ def test_advisor_tool_result_preserved_in_response():
{"type": "text", "text": "Here is the implementation."},
]
}
- text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content(
- completion_response
- )
+ text, _, _, _, tool_calls, _, tool_results, _ = config.extract_response_content(completion_response)
assert "Consulting advisor." in text
assert "Here is the implementation." in text
# server_tool_use (advisor) should be a tool_call
@@ -4843,9 +4754,7 @@ def test_basic_sanitize_anthropic_tool_name_replaces_invalid_chars():
)
assert (
- _basic_sanitize_anthropic_tool_name(
- "github_openapi_mcp-actions/download-job-logs-for-workflow-run"
- )
+ _basic_sanitize_anthropic_tool_name("github_openapi_mcp-actions/download-job-logs-for-workflow-run")
== "github_openapi_mcp-actions_download-job-logs-for-workflow-run"
)
# other punctuation
@@ -4874,9 +4783,7 @@ def test_build_anthropic_tool_name_maps_no_collisions():
]
)
assert forward == {
- "actions/download-job-logs-for-workflow-run": (
- "actions_download-job-logs-for-workflow-run"
- ),
+ "actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run"),
"pulls/list-files": "pulls_list-files",
}
assert reverse == {v: k for k, v in forward.items()}
@@ -4927,9 +4834,7 @@ def test_build_anthropic_tool_name_maps_three_way_collision():
_build_anthropic_tool_name_maps,
)
- forward, reverse = _build_anthropic_tool_name_maps(
- ["foo_bar", "foo/bar", "foo.bar"]
- )
+ forward, reverse = _build_anthropic_tool_name_maps(["foo_bar", "foo/bar", "foo.bar"])
assert "foo_bar" not in forward # untouched
assert forward["foo/bar"] == "foo_bar_2"
assert forward["foo.bar"] == "foo_bar_3"
@@ -5002,16 +4907,13 @@ def test_map_openai_params_does_not_pollute_optional_params_with_internal_keys()
)
# No internal keys may appear in optional_params for ANY input.
for key in optional_params:
- assert not key.startswith(
- "_anthropic_tool_name"
- ), f"optional_params leaked internal key {key!r}: {optional_params}"
+ assert not key.startswith("_anthropic_tool_name"), (
+ f"optional_params leaked internal key {key!r}: {optional_params}"
+ )
# And no key starting with `_` either; optional_params should only
# contain documented Anthropic Messages API parameters.
for key in optional_params:
- assert not key.startswith("_"), (
- f"optional_params leaked underscore-prefixed key {key!r}: "
- f"{optional_params}"
- )
+ assert not key.startswith("_"), f"optional_params leaked underscore-prefixed key {key!r}: {optional_params}"
def test_map_openai_params_no_maps_when_all_names_already_valid():
@@ -5040,11 +4942,7 @@ def test_map_openai_params_no_maps_when_all_names_already_valid():
def test_rewrite_tool_names_in_messages_uses_forward_map():
config = AnthropicConfig()
- forward_map = {
- "actions/download-job-logs-for-workflow-run": (
- "actions_download-job-logs-for-workflow-run"
- )
- }
+ forward_map = {"actions/download-job-logs-for-workflow-run": ("actions_download-job-logs-for-workflow-run")}
messages = [
{"role": "user", "content": "go"},
{
@@ -5067,15 +4965,9 @@ def test_rewrite_tool_names_in_messages_uses_forward_map():
out = config._rewrite_tool_names_in_messages(messages, forward_map)
# input list must not be mutated
- assert (
- messages[1]["tool_calls"][0]["function"]["name"]
- == "actions/download-job-logs-for-workflow-run"
- )
+ assert messages[1]["tool_calls"][0]["function"]["name"] == "actions/download-job-logs-for-workflow-run"
# output rewritten according to forward map
- assert (
- out[1]["tool_calls"][0]["function"]["name"]
- == "actions_download-job-logs-for-workflow-run"
- )
+ assert out[1]["tool_calls"][0]["function"]["name"] == "actions_download-job-logs-for-workflow-run"
# non-tool-call messages pass through unchanged (same object)
assert out[0] is messages[0]
assert out[2] is messages[2]
@@ -5151,9 +5043,7 @@ def test_sanitize_tool_names_in_request_does_not_mutate_caller_tool_dicts():
caller_tools = [caller_tool]
optional_params: dict = {"tools": caller_tools}
- forward, reverse = config._sanitize_tool_names_in_request(
- optional_params=optional_params
- )
+ forward, reverse = config._sanitize_tool_names_in_request(optional_params=optional_params)
assert forward.get(original_name)
sanitized = forward[original_name]
@@ -5302,10 +5192,7 @@ def test_streaming_iterator_reverse_maps_tool_use_name():
parsed = iterator.chunk_parser(chunk=chunk)
tool_calls = parsed.choices[0].delta.tool_calls
assert tool_calls is not None and len(tool_calls) == 1
- assert (
- tool_calls[0]["function"]["name"]
- == "actions/download-job-logs-for-workflow-run"
- )
+ assert tool_calls[0]["function"]["name"] == "actions/download-job-logs-for-workflow-run"
def test_streaming_iterator_passthrough_when_name_not_in_map():
@@ -5401,9 +5288,9 @@ def test_transform_request_does_not_leak_internal_keys_into_body():
for tool in data.get("tools", []):
name = tool.get("name")
assert isinstance(name, str)
- assert _re.fullmatch(
- r"[a-zA-Z0-9_-]{1,128}", name
- ), f"sanitized tool name {name!r} still violates Anthropic regex"
+ assert _re.fullmatch(r"[a-zA-Z0-9_-]{1,128}", name), (
+ f"sanitized tool name {name!r} still violates Anthropic regex"
+ )
# Sent name for the bad tool is the disambiguated form, valid name passes through.
sent_names = {t["name"] for t in data["tools"]}
@@ -5539,9 +5426,7 @@ def test_transform_request_rewrites_tool_names_in_history():
for block in content:
if isinstance(block, dict) and block.get("type") == "tool_use":
tool_use_names.append(block.get("name"))
- assert (
- tool_use_names
- ), "expected at least one tool_use block in transformed messages"
+ assert tool_use_names, "expected at least one tool_use block in transformed messages"
for name in tool_use_names:
assert name == "actions_download-job-logs-for-workflow-run", (
f"history tool_use.name {name!r} not rewritten -- Anthropic will "
@@ -5565,19 +5450,12 @@ def test_sanitize_tool_names_in_request_skips_hosted_tools():
}
forward, reverse = AnthropicConfig._sanitize_tool_names_in_request(optional_params)
# Only the custom tool was rewritten.
- assert forward == {
- "actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run"
- }
- assert reverse == {
- "actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run"
- }
+ assert forward == {"actions/download-job-logs-for-workflow-run": "actions_download-job-logs-for-workflow-run"}
+ assert reverse == {"actions_download-job-logs-for-workflow-run": "actions/download-job-logs-for-workflow-run"}
# Hosted tool's name unchanged.
assert optional_params["tools"][0]["name"] == "web_search"
# Custom tool's name updated in place.
- assert (
- optional_params["tools"][1]["name"]
- == "actions_download-job-logs-for-workflow-run"
- )
+ assert optional_params["tools"][1]["name"] == "actions_download-job-logs-for-workflow-run"
def test_sanitize_tool_names_in_request_no_tools_is_noop():
@@ -5811,9 +5689,7 @@ def test_translate_system_message_keeps_billing_header_for_first_party_anthropic
assert config.should_strip_billing_metadata() is False
result = config.translate_system_message(
- messages=_system_with_billing_header(
- "You are Claude Code, Anthropic's official CLI for Claude."
- )
+ messages=_system_with_billing_header("You are Claude Code, Anthropic's official CLI for Claude.")
)
texts = [block["text"] for block in result]
@@ -5829,9 +5705,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock():
config = BedrockClaudePlatformConfig()
assert config.should_strip_billing_metadata() is True
- result = config.translate_system_message(
- messages=_system_with_billing_header("real system prompt")
- )
+ result = config.translate_system_message(messages=_system_with_billing_header("real system prompt"))
texts = [block["text"] for block in result]
assert all(not t.startswith("x-anthropic-billing-header:") for t in texts)
@@ -5897,9 +5771,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke():
config = AmazonAnthropicClaudeConfig()
assert config.should_strip_billing_metadata() is True
- result = config.translate_system_message(
- messages=_system_with_billing_header("real system prompt")
- )
+ result = config.translate_system_message(messages=_system_with_billing_header("real system prompt"))
texts = [block["text"] for block in result]
assert all(not t.startswith("x-anthropic-billing-header:") for t in texts)
@@ -5953,9 +5825,7 @@ def test_translate_system_message_strips_billing_header_for_bedrock_invoke():
),
],
)
-def test_should_strip_billing_metadata_by_provider(
- module_path, class_name, expected_strip
-):
+def test_should_strip_billing_metadata_by_provider(module_path, class_name, expected_strip):
import importlib
config_cls = getattr(importlib.import_module(module_path), class_name)
@@ -6127,12 +5997,8 @@ def test_sampling_param_gating_driven_by_model_map_flag(monkeypatch):
"""The drop/raise decision must come from ``supports_sampling_params`` in
the model map, not just name matching: a flagged entry gates a model whose
name says nothing, and an explicit ``true`` overrides the name fallback."""
- monkeypatch.setitem(
- litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False}
- )
- monkeypatch.setitem(
- litellm.model_cost, "claude-fable-5-test", {"supports_sampling_params": True}
- )
+ monkeypatch.setitem(litellm.model_cost, "claude-zeta-9", {"supports_sampling_params": False})
+ monkeypatch.setitem(litellm.model_cost, "claude-fable-5-test", {"supports_sampling_params": True})
config = AnthropicConfig()
flagged_off = config.map_openai_params(
@@ -6252,9 +6118,7 @@ def test_is_anthropic_usage_object_rejects_responses_api_usage():
("claude-sonnet-4-5-20250929", False),
],
)
-def test_disabled_thinking_omitted_only_for_always_on_models(
- local_model_cost_map, model, expected_dropped
-):
+def test_disabled_thinking_omitted_only_for_always_on_models(local_model_cost_map, model, expected_dropped):
"""``thinking={"type": "disabled"}`` is omitted for always-on-thinking models
(Fable/Mythos, which 400 on it: the API remedy is to omit the param) and is
forwarded verbatim for every model that accepts it."""
@@ -6300,9 +6164,7 @@ def test_forced_tool_choice_raises_clean_error_on_fable_5_1_without_drop_params(
"tool_choice",
["required", {"type": "required"}, {"type": "function", "function": {"name": "get_weather"}}],
)
-def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params(
- local_model_cost_map, tool_choice
-):
+def test_forced_tool_choice_downgraded_to_auto_on_fable_5_1_with_drop_params(local_model_cost_map, tool_choice):
config = AnthropicConfig()
result = config.map_openai_params(
@@ -6329,9 +6191,7 @@ def test_forced_tool_choice_downgrade_keeps_parallel_tool_calls_flag(local_model
@pytest.mark.parametrize("tool_choice, expected_type", [("auto", "auto"), ("none", "none")])
-def test_unforced_tool_choice_forwarded_on_fable_5_1(
- local_model_cost_map, tool_choice, expected_type, monkeypatch
-):
+def test_unforced_tool_choice_forwarded_on_fable_5_1(local_model_cost_map, tool_choice, expected_type, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
config = AnthropicConfig()
@@ -6346,9 +6206,7 @@ def test_unforced_tool_choice_forwarded_on_fable_5_1(
@pytest.mark.parametrize("model", ["claude-fable-5", "claude-opus-5", "claude-sonnet-5"])
-def test_forced_tool_choice_forwarded_on_models_that_support_it(
- local_model_cost_map, model, monkeypatch
-):
+def test_forced_tool_choice_forwarded_on_models_that_support_it(local_model_cost_map, model, monkeypatch):
monkeypatch.setattr(litellm, "drop_params", False)
config = AnthropicConfig()
@@ -6519,3 +6377,424 @@ def test_eager_input_streaming_reaches_anthropic_request_tools():
assert result["tools"][0]["eager_input_streaming"] is True
assert result["tools"][0]["name"] == "write_file"
+
+
+# ---------------------------------------------------------------------------
+# Mid-conversation ``role: "system"`` on the chat completions path.
+#
+# Hoisting a later system message into the top-level ``system`` block rewrites
+# the cached prefix and re-bills the whole conversation at cache-write pricing
+# on every reminder (#36559). The chat path must keep the prefix stable: leading
+# system messages still become the ``system`` param, later ones stay in place as
+# ``role: "system"`` on models flagged ``supports_mid_conversation_system`` and
+# become a user turn on models that reject the role inside ``messages``.
+# ---------------------------------------------------------------------------
+
+UNFLAGGED_CLAUDE = "claude-opus-4-7"
+FLAGGED_CLAUDE = "claude-opus-4-8"
+CONVERTED_SYSTEM_NOTE = (
+ "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."
+)
+REMINDER_TEXT = "Answer with exactly one word."
+CACHED_SYSTEM_BLOCK = {"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}
+
+
+def _chat_request(config: AnthropicConfig, model: str, messages: list[dict]) -> dict:
+ return config.transform_request(
+ model=model,
+ messages=messages,
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+
+def _reminder_conversation() -> list[dict]:
+ """The shape Claude Code sends mid-session: cached system prompt, turns, a
+ reminder right after a user turn, an assistant turn, a fresh user turn."""
+ return [
+ {"role": "system", "content": [dict(CACHED_SYSTEM_BLOCK)]},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ {"role": "system", "content": REMINDER_TEXT},
+ {"role": "assistant", "content": "Second answer"},
+ {"role": "user", "content": "Third question"},
+ ]
+
+
+def _texts(message: dict) -> list[str]:
+ return [block["text"] for block in message["content"] if block.get("type") == "text"]
+
+
+def test_chat_unflagged_model_converts_mid_conversation_system_to_user_turn(local_model_cost_map):
+ result = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, _reminder_conversation())
+
+ assert result["system"] == [CACHED_SYSTEM_BLOCK]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "assistant", "user"]
+ assert _texts(result["messages"][2]) == ["Second question", CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
+
+
+def test_chat_flagged_model_keeps_mid_conversation_system_in_messages(local_model_cost_map):
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, _reminder_conversation())
+
+ assert result["system"] == [CACHED_SYSTEM_BLOCK]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
+ assert result["messages"][3] == {"role": "system", "content": [{"type": "text", "text": REMINDER_TEXT}]}
+
+
+def test_chat_flagged_model_keeps_cache_control_on_mid_conversation_system(local_model_cost_map):
+ messages = _reminder_conversation()
+ messages[4] = {
+ "role": "system",
+ "content": [{"type": "text", "text": REMINDER_TEXT, "cache_control": {"type": "ephemeral"}}],
+ }
+
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
+
+ assert result["messages"][3]["content"] == [
+ {"type": "text", "text": REMINDER_TEXT, "cache_control": {"type": "ephemeral"}}
+ ]
+
+
+def test_chat_flagged_model_moves_system_after_the_user_turn_it_precedes(local_model_cost_map):
+ """Anthropic only accepts role=system directly after a user turn; an
+ OpenAI-shaped client that puts the reminder before its next question gets a
+ placement-valid request without the reminder leaving ``messages``."""
+ messages = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "system", "content": REMINDER_TEXT},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
+
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "system"]
+ assert _texts(result["messages"][2]) == ["Second question"]
+ assert _texts(result["messages"][3]) == [REMINDER_TEXT]
+
+
+def test_chat_flagged_model_converts_system_with_no_following_user_turn(local_model_cost_map):
+ messages = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "system", "content": REMINDER_TEXT},
+ ]
+
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
+
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user"]
+ assert _texts(result["messages"][2]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
+
+
+@pytest.mark.parametrize(
+ "empty_content",
+ [[], None, [{"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}]],
+ ids=["empty-list", "none", "unsupported-part-only"],
+)
+def test_chat_flagged_model_converts_a_system_behind_a_user_turn_that_sends_nothing(
+ local_model_cost_map, empty_content
+):
+ messages = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": empty_content},
+ {"role": "system", "content": REMINDER_TEXT},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
+
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user"]
+ assert _texts(result["messages"][0]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
+
+
+USER_PART_BY_TYPE = {
+ "text": {"type": "text", "text": "hello"},
+ "image_url": {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}},
+ "document": {"type": "document", "source": {"type": "text", "media_type": "text/plain", "data": "hello"}},
+ "file": {"type": "file", "file": {"file_data": "data:text/plain;base64,aGVsbG8=", "filename": "hello.txt"}},
+ "input_audio": {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}},
+ "video_url": {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}},
+}
+
+
+@pytest.mark.parametrize("part_type", sorted(USER_PART_BY_TYPE))
+def test_chat_flagged_model_anchors_a_system_on_a_user_turn_exactly_when_that_turn_reaches_the_wire(
+ local_model_cost_map, part_type
+):
+ part_only_turn = {"role": "user", "content": [USER_PART_BY_TYPE[part_type]]}
+ tail = [{"role": "assistant", "content": "First answer"}, {"role": "user", "content": "Second question"}]
+
+ without_reminder = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, [part_only_turn, *tail])
+ with_reminder = _chat_request(
+ AnthropicConfig(), FLAGGED_CLAUDE, [part_only_turn, {"role": "system", "content": REMINDER_TEXT}, *tail]
+ )
+
+ turn_reaches_wire = [m["role"] for m in without_reminder["messages"]] == ["user", "assistant", "user"]
+ expected_roles = ["user", "system", "assistant", "user"] if turn_reaches_wire else ["user", "assistant", "user"]
+ assert [m["role"] for m in with_reminder["messages"]] == expected_roles
+
+
+ASSISTANT_TURN_BY_SHAPE = {
+ "text": {"role": "assistant", "content": "First answer"},
+ "tool-calls": {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "toolu_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
+ },
+ "signed-thinking-part": {
+ "role": "assistant",
+ "content": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+ "empty-string": {"role": "assistant", "content": ""},
+ "whitespace-string": {"role": "assistant", "content": " "},
+ "empty-text-part": {"role": "assistant", "content": [{"type": "text", "text": ""}]},
+ "none": {"role": "assistant", "content": None},
+ "empty-list": {"role": "assistant", "content": []},
+ "unsigned-thinking-part": {"role": "assistant", "content": [{"type": "thinking", "thinking": "hm"}]},
+ "signed-thinking-block": {
+ "role": "assistant",
+ "content": None,
+ "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+ "redacted-thinking-block": {
+ "role": "assistant",
+ "content": None,
+ "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}],
+ },
+ "encrypted-thinking-part": {
+ "role": "assistant",
+ "content": [{"type": "thinking", "thinking": "hm", "signature": encrypted_reasoning_signature("abc")}],
+ },
+ "encrypted-redacted-thinking-block": {
+ "role": "assistant",
+ "content": None,
+ "thinking_blocks": [{"type": "redacted_thinking", "data": encrypted_reasoning_signature("abc")}],
+ },
+ "unsigned-inline-part-hides-signed-block": {
+ "role": "assistant",
+ "content": [{"type": "thinking", "thinking": "hm"}],
+ "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+ "inline-redacted-part-hides-redacted-block": {
+ "role": "assistant",
+ "content": [{"type": "redacted_thinking", "data": "x"}],
+ "thinking_blocks": [{"type": "redacted_thinking", "data": "x"}],
+ },
+ "text-part-beside-signed-block": {
+ "role": "assistant",
+ "content": [{"type": "text", "text": "First answer"}],
+ "thinking_blocks": [{"type": "thinking", "thinking": "hm", "signature": "s"}],
+ },
+}
+
+
+@pytest.mark.parametrize("shape", sorted(ASSISTANT_TURN_BY_SHAPE))
+def test_chat_flagged_model_keeps_a_system_exactly_when_the_assistant_turn_after_it_reaches_the_wire(
+ local_model_cost_map, shape
+):
+ first_turn = {"role": "user", "content": "First question"}
+ tail = [ASSISTANT_TURN_BY_SHAPE[shape], {"role": "user", "content": "Second question"}]
+
+ without_reminder = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, [first_turn, *tail])
+ with_reminder = _chat_request(
+ AnthropicConfig(), FLAGGED_CLAUDE, [first_turn, {"role": "system", "content": REMINDER_TEXT}, *tail]
+ )
+
+ turn_reaches_wire = [m["role"] for m in without_reminder["messages"]] == ["user", "assistant", "user"]
+ expected_roles = ["user", "system", "assistant", "user"] if turn_reaches_wire else ["user", "user"]
+ assert [m["role"] for m in with_reminder["messages"]] == expected_roles
+ if not turn_reaches_wire:
+ assert _texts(with_reminder["messages"][0]) == ["First question", CONVERTED_SYSTEM_NOTE, REMINDER_TEXT]
+
+
+def test_chat_flagged_model_merges_adjacent_system_messages(local_model_cost_map):
+ messages = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "First question"},
+ {"role": "system", "content": "Reminder one."},
+ {"role": "system", "content": "Reminder two."},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ ]
+
+ result = _chat_request(AnthropicConfig(), FLAGGED_CLAUDE, messages)
+
+ assert [m["role"] for m in result["messages"]] == ["user", "system", "assistant", "user"]
+ assert _texts(result["messages"][1]) == ["Reminder one.", "Reminder two."]
+
+
+def test_chat_unflagged_model_keeps_tool_result_first_when_system_precedes_tool_message(local_model_cost_map):
+ messages = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "Weather?"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "get_weather", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "system", "content": REMINDER_TEXT},
+ {"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
+ {"role": "user", "content": "Thanks"},
+ ]
+
+ result = _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, messages)
+
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user"]
+ blocks = result["messages"][2]["content"]
+ assert blocks[0]["type"] == "tool_result"
+ assert blocks[0]["tool_use_id"] == "call_1"
+ assert _texts(result["messages"][2]) == [CONVERTED_SYSTEM_NOTE, REMINDER_TEXT, "Thanks"]
+
+
+def test_chat_transform_request_does_not_mutate_caller_messages(local_model_cost_map):
+ messages = _reminder_conversation()
+ snapshot = copy.deepcopy(messages)
+
+ _chat_request(AnthropicConfig(), UNFLAGGED_CLAUDE, messages)
+
+ assert messages == snapshot
+
+
+_CHAT_CONFIGS = [
+ pytest.param(AnthropicConfig, UNFLAGGED_CLAUDE, id="anthropic-unflagged"),
+ pytest.param(AnthropicConfig, FLAGGED_CLAUDE, id="anthropic-flagged"),
+ pytest.param(VertexAIAnthropicConfig, UNFLAGGED_CLAUDE, id="vertex_ai-unflagged"),
+ pytest.param(VertexAIAnthropicConfig, FLAGGED_CLAUDE, id="vertex_ai-flagged"),
+ pytest.param(AzureAnthropicConfig, UNFLAGGED_CLAUDE, id="azure_ai-unflagged"),
+ pytest.param(AzureAnthropicConfig, FLAGGED_CLAUDE, id="azure_ai-flagged"),
+ pytest.param(AmazonAnthropicClaudeConfig, "invoke/us.anthropic.claude-opus-4-7", id="bedrock_invoke-unflagged"),
+ pytest.param(AmazonAnthropicClaudeConfig, "invoke/us.anthropic.claude-opus-4-8", id="bedrock_invoke-flagged"),
+]
+
+
+@pytest.mark.parametrize("config_cls, model", _CHAT_CONFIGS)
+def test_chat_mid_conversation_system_keeps_earlier_turns_a_prefix_of_the_next_request(
+ local_model_cost_map, config_cls, model
+):
+ """The provider-side prompt cache is a prefix match over ``system`` +
+ ``messages``. Whatever the policy for the reminder, turn N's request must
+ stay a prefix of turn N+1's request or the whole conversation is re-billed.
+
+ Anthropic combines consecutive same-role messages into one turn, so the
+ cache-relevant sequence is ``(role, content block)`` pairs, not the message
+ list: a reminder that joins the preceding user turn still extends the prefix.
+ """
+ conversation = _reminder_conversation()
+
+ earlier = _chat_request(config_cls(), model, copy.deepcopy(conversation[:4]))
+ later = _chat_request(config_cls(), model, copy.deepcopy(conversation))
+
+ assert later["system"] == earlier["system"]
+ earlier_blocks = _role_block_pairs(earlier["messages"])
+ later_blocks = _role_block_pairs(later["messages"])
+ assert later_blocks[: len(earlier_blocks)] == earlier_blocks
+ assert len(later_blocks) > len(earlier_blocks)
+
+
+def _role_block_pairs(messages: list[dict]) -> list[tuple[str, object]]:
+ return [
+ (message["role"], block)
+ for message in messages
+ for block in (message["content"] if isinstance(message["content"], list) else [message["content"]])
+ ]
+
+
+def _thinking_reply(text: str) -> dict:
+ return {
+ "role": "assistant",
+ "content": text,
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": f"sig-{text}"}],
+ }
+
+
+def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], list[dict], list[dict]]:
+ turn_n = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "First question"}]
+ reminder = {"role": "system", "content": REMINDER_TEXT}
+ second_question = {"role": "user", "content": "Second question"}
+ second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question]
+ turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn]
+ turn_n_plus_two = [
+ *turn_n_plus_one,
+ _thinking_reply("Second answer"),
+ {"role": "user", "content": "Third question"},
+ ]
+ return turn_n, turn_n_plus_one, turn_n_plus_two
+
+
+def _replayed_prefix(request: dict, message_count: int) -> str:
+ replayed = {
+ "system": request.get("system"),
+ "tools": request.get("tools"),
+ "messages": request["messages"][:message_count],
+ }
+ return json.dumps(replayed, sort_keys=True)
+
+
+def _assert_prefix_stable(requests: list[dict]) -> None:
+ for earlier, later in zip(requests, requests[1:]):
+ count = len(earlier["messages"])
+ assert _replayed_prefix(later, count) == _replayed_prefix(earlier, count)
+
+
+@pytest.mark.parametrize("reminder_after_user", [True, False])
+def test_chat_flagged_model_replays_a_byte_identical_prefix_around_a_mid_conversation_reminder(
+ local_model_cost_map, reminder_after_user
+):
+ """Preserved thinking binds each signed block to the request prefix it was created
+ under (``system``, ``tools`` and the earlier messages), so turn N's transformed
+ request must be a byte-identical prefix of turn N+1's or the block is dropped."""
+ requests = [
+ AnthropicConfig().transform_request(
+ model="claude-fable-5-1", messages=copy.deepcopy(turn), optional_params={}, litellm_params={}, headers={}
+ )
+ for turn in _preserved_thinking_turns(reminder_after_user)
+ ]
+
+ _assert_prefix_stable(requests)
+ assert [m["role"] for m in requests[1]["messages"]] == ["user", "assistant", "user", "system"]
+ assert [m["role"] for m in requests[2]["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
+
+
+def test_chat_dummy_tool_result_for_an_orphaned_tool_call_replays_a_byte_identical_prefix(
+ local_model_cost_map, monkeypatch
+):
+ monkeypatch.setattr(litellm, "modify_params", True)
+ tools = [
+ {"name": "lookup", "description": "Look something up", "input_schema": {"type": "object", "properties": {}}}
+ ]
+ orphaned_call = {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
+ }
+ turn_n = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "First question"},
+ orphaned_call,
+ ]
+ turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), {"role": "user", "content": "Second question"}]
+ requests = [
+ AnthropicConfig().transform_request(
+ model="claude-fable-5-1",
+ messages=copy.deepcopy(turn),
+ optional_params={"tools": copy.deepcopy(tools)},
+ litellm_params={},
+ headers={},
+ )
+ for turn in (turn_n, turn_n_plus_one)
+ ]
+
+ _assert_prefix_stable(requests)
+ assert [m["role"] for m in requests[0]["messages"]] == ["user", "assistant", "user"]
+ assert requests[0]["messages"][2]["content"][0]["type"] == "tool_result"
diff --git a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py
index 9e2bfb08852..ddbc168589a 100644
--- a/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py
+++ b/tests/test_litellm/llms/azure_ai/claude/test_azure_anthropic_transformation.py
@@ -437,3 +437,51 @@ class TestAzureAnthropicConfig:
assert "anthropic-beta" in headers
assert "compact-2026-01-12" in headers["anthropic-beta"]
assert "context-management-2025-06-27" in headers["anthropic-beta"]
+
+
+def _mid_conversation_system_conversation() -> list[dict]:
+ return [
+ {"role": "system", "content": [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ {"role": "system", "content": "Answer with exactly one word."},
+ {"role": "assistant", "content": "Second answer"},
+ {"role": "user", "content": "Third question"},
+ ]
+
+
+def test_chat_unflagged_model_converts_mid_conversation_system_instead_of_hoisting(local_model_cost_map):
+ """A hoisted reminder rewrites the top-level system block and invalidates the
+ prompt cache for the whole conversation (#36559)."""
+ result = AzureAnthropicConfig().transform_request(
+ model="claude-opus-4-7",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "assistant", "user"]
+ texts = [b["text"] for b in result["messages"][2]["content"] if b.get("type") == "text"]
+ assert texts[0] == "Second question"
+ assert texts[-1] == "Answer with exactly one word."
+
+
+def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_model_cost_map):
+ result = AzureAnthropicConfig().transform_request(
+ model="claude-opus-4-8",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
+ assert result["messages"][3] == {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer with exactly one word."}],
+ }
+
diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
index ad77f9d4d1b..499096621c5 100644
--- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
+++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
@@ -1,3 +1,4 @@
+import copy
import json
import os
from typing import Final
@@ -8,6 +9,7 @@ import pytest
import litellm
from litellm import ModelResponse
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import CONVERTED_SYSTEM_NOTE
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
from litellm.types.llms.bedrock import ConverseTokenUsageBlock
@@ -7584,6 +7586,246 @@ def test_eager_input_streaming_non_boolean_is_a_bad_request():
)
+def test_mid_conversation_system_after_multiple_tool_results():
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "calling tools",
+ "tool_calls": [
+ {
+ "id": "call_a",
+ "type": "function",
+ "function": {"name": "f", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "system", "content": "reminder"},
+ {"role": "tool", "tool_call_id": "call_a", "content": "r1"},
+ {"role": "tool", "tool_call_id": "call_b", "content": "r2"},
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert system_blocks == []
+ assert [m["role"] for m in out_messages] == [
+ "user",
+ "assistant",
+ "tool",
+ "tool",
+ "user",
+ "user",
+ ]
+ assert out_messages[2]["content"] == "r1"
+ assert out_messages[3]["content"] == "r2"
+ # Reminder lands after ALL tool results, not between them.
+ assert out_messages[4]["content"][1]["text"] == "reminder"
+ assert out_messages[5]["content"] == "done"
+
+
+def test_mid_conversation_system_reorders_around_a_pydantic_assistant_tool_call():
+ config = AmazonConverseConfig()
+ assistant = litellm.Message(
+ role="assistant",
+ content="calling tools",
+ tool_calls=[{"id": "call_a", "type": "function", "function": {"name": "f", "arguments": "{}"}}],
+ )
+ messages = [
+ {"role": "user", "content": "hi"},
+ assistant,
+ {"role": "system", "content": "reminder"},
+ {"role": "tool", "tool_call_id": "call_a", "content": "r1"},
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert system_blocks == []
+ assert [m["role"] for m in out_messages] == ["user", "assistant", "tool", "user", "user"]
+ assert out_messages[1] is assistant
+ assert out_messages[3]["content"][1]["text"] == "reminder"
+
+
+def test_mid_conversation_multi_system_run_after_multiple_tool_results():
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "assistant",
+ "content": "calling tools",
+ "tool_calls": [
+ {
+ "id": "call_a",
+ "type": "function",
+ "function": {"name": "f", "arguments": "{}"},
+ }
+ ],
+ },
+ {"role": "system", "content": "reminder 1"},
+ {"role": "system", "content": "reminder 2"},
+ {"role": "tool", "tool_call_id": "call_a", "content": "r1"},
+ {"role": "tool", "tool_call_id": "call_b", "content": "r2"},
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, _ = config._transform_system_message(messages)
+ assert [m["role"] for m in out_messages] == [
+ "user",
+ "assistant",
+ "tool",
+ "tool",
+ "user",
+ "user",
+ "user",
+ ]
+ assert out_messages[4]["content"][1]["text"] == "reminder 1"
+ assert out_messages[5]["content"][1]["text"] == "reminder 2"
+
+
+def test_opens_with_tool_result_rejects_non_dict():
+ config = AmazonConverseConfig()
+ assert config._opens_with_tool_result("not-a-dict") is False
+ assert config._opens_with_tool_result(None) is False
+ assert config._opens_with_tool_result([{"role": "tool"}]) is False
+
+
+def test_mid_conversation_system_without_tools_stays_in_place():
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "hi"},
+ {"role": "assistant", "content": "hello"},
+ {"role": "system", "content": "reminder"},
+ {"role": "user", "content": "thanks"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert [b["text"] for b in system_blocks if "text" in b] == ["You are helpful."]
+ assert [m["role"] for m in out_messages] == ["user", "assistant", "user", "user"]
+ assert out_messages[2]["content"][1]["text"] == "reminder"
+ assert out_messages[3]["content"] == "thanks"
+
+
+def test_mid_conversation_system_str_with_cache_control():
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "system",
+ "content": "reminder",
+ "cache_control": {"type": "ephemeral"},
+ },
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert system_blocks == []
+ assert out_messages[1]["role"] == "user"
+ assert out_messages[1]["content"][1] == {
+ "type": "text",
+ "text": "reminder",
+ "cache_control": {"type": "ephemeral"},
+ }
+
+
+def test_mid_conversation_system_list_content_with_cache_control():
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "user", "content": "hi"},
+ {
+ "role": "system",
+ "content": [
+ {"type": "text", "text": "keep this", "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": "plain"},
+ {"type": "text", "text": ""},
+ {"type": "image", "source": "x"},
+ "raw-string",
+ ],
+ },
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert system_blocks == []
+ blocks = out_messages[1]["content"]
+ assert blocks[0]["text"] == CONVERTED_SYSTEM_NOTE
+ assert blocks[1] == {
+ "type": "text",
+ "text": "keep this",
+ "cache_control": {"type": "ephemeral"},
+ }
+ assert blocks[2] == {"type": "text", "text": "plain"}
+ assert len(blocks) == 3
+
+
+@pytest.mark.parametrize(
+ "empty_content",
+ ["", [], None, [{"type": "image", "source": "x"}, {"type": "text", "text": ""}]],
+ ids=["empty-string", "empty-list", "none", "no-text-parts"],
+)
+def test_mid_conversation_system_entry_without_text_is_dropped(empty_content):
+ config = AmazonConverseConfig()
+ messages = [
+ {"role": "user", "content": "hi"},
+ {"role": "system", "content": empty_content},
+ {"role": "user", "content": "done"},
+ ]
+ out_messages, system_blocks = config._transform_system_message(messages)
+ assert system_blocks == []
+ assert out_messages == [{"role": "user", "content": "hi"}, {"role": "user", "content": "done"}]
+
+
+def _thinking_reply(text: str) -> dict:
+ return {
+ "role": "assistant",
+ "content": text,
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": f"sig-{text}"}],
+ }
+
+
+def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], list[dict], list[dict]]:
+ turn_n = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "First question"}]
+ reminder = {"role": "system", "content": "Answer with exactly one word."}
+ second_question = {"role": "user", "content": "Second question"}
+ second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question]
+ turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn]
+ turn_n_plus_two = [*turn_n_plus_one, _thinking_reply("Second answer"), {"role": "user", "content": "Third question"}]
+ return turn_n, turn_n_plus_one, turn_n_plus_two
+
+
+def _replayed_prefix(request: dict, message_count: int) -> str:
+ replayed = {
+ "system": request.get("system"),
+ "toolConfig": request.get("toolConfig"),
+ "messages": request["messages"][:message_count],
+ }
+ return json.dumps(replayed, sort_keys=True)
+
+
+def _assert_prefix_stable(requests: list[dict]) -> None:
+ for earlier, later in zip(requests, requests[1:]):
+ count = len(earlier["messages"])
+ assert _replayed_prefix(later, count) == _replayed_prefix(earlier, count)
+
+
+@pytest.mark.parametrize("reminder_after_user", [True, False])
+def test_flagged_model_replays_a_byte_identical_prefix_around_a_mid_conversation_reminder(
+ local_model_cost_map, reminder_after_user
+):
+ """Converse rejects ``role: system`` inside ``messages``, so the reminder becomes a
+ user turn in place; hoisting it into ``system`` would change the prefix every
+ signed thinking block in the history is bound to."""
+ requests = [
+ AmazonConverseConfig().transform_request(
+ model="bedrock/us.anthropic.claude-fable-5-1",
+ messages=copy.deepcopy(turn),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+ for turn in _preserved_thinking_turns(reminder_after_user)
+ ]
+
+ _assert_prefix_stable(requests)
+ assert requests[1]["system"] == [{"text": "You are terse."}]
+ assert [m["role"] for m in requests[1]["messages"]] == ["user", "assistant", "user"]
+ assert [m["role"] for m in requests[2]["messages"]] == ["user", "assistant", "user", "assistant", "user"]
+
+
@pytest.mark.parametrize("model", ("anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7"))
def test_converse_accepts_anthropic_default_temperature(model: str) -> None:
result: Final = litellm.utils.get_optional_params(
diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
index 37a619d6400..ca4a3dedb4a 100644
--- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
+++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py
@@ -1,4 +1,7 @@
+import copy
+import json
+
import pytest
from litellm.anthropic_beta_headers_manager import (
@@ -771,3 +774,104 @@ def test_vertex_ai_anthropic_tool_based_response_format_still_upgrades_legacy_th
assert "tools" in result_params
assert result_params["thinking"] == {"type": "adaptive"}
assert result_params["output_config"] == {"effort": "high"}
+
+
+
+
+def _mid_conversation_system_conversation() -> list[dict]:
+ return [
+ {"role": "system", "content": [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ {"role": "system", "content": "Answer with exactly one word."},
+ {"role": "assistant", "content": "Second answer"},
+ {"role": "user", "content": "Third question"},
+ ]
+
+
+def test_chat_unflagged_model_converts_mid_conversation_system_instead_of_hoisting(local_model_cost_map):
+ """A hoisted reminder rewrites the top-level system block and invalidates the
+ prompt cache for the whole conversation (#36559)."""
+ result = VertexAIAnthropicConfig().transform_request(
+ model="claude-opus-4-7",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "assistant", "user"]
+ texts = [b["text"] for b in result["messages"][2]["content"] if b.get("type") == "text"]
+ assert texts[0] == "Second question"
+ assert texts[-1] == "Answer with exactly one word."
+
+
+def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_model_cost_map):
+ result = VertexAIAnthropicConfig().transform_request(
+ model="claude-opus-4-8",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
+ assert result["messages"][3] == {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer with exactly one word."}],
+ }
+
+
+def _thinking_reply(text: str) -> dict:
+ return {
+ "role": "assistant",
+ "content": text,
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": f"sig-{text}"}],
+ }
+
+
+def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], list[dict], list[dict]]:
+ turn_n = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "First question"}]
+ reminder = {"role": "system", "content": "Answer with exactly one word."}
+ second_question = {"role": "user", "content": "Second question"}
+ second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question]
+ turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn]
+ turn_n_plus_two = [*turn_n_plus_one, _thinking_reply("Second answer"), {"role": "user", "content": "Third question"}]
+ return turn_n, turn_n_plus_one, turn_n_plus_two
+
+
+def _replayed_prefix(request: dict, message_count: int) -> str:
+ replayed = {
+ "system": request.get("system"),
+ "tools": request.get("tools"),
+ "messages": request["messages"][:message_count],
+ }
+ return json.dumps(replayed, sort_keys=True)
+
+
+def _assert_prefix_stable(requests: list[dict]) -> None:
+ for earlier, later in zip(requests, requests[1:]):
+ count = len(earlier["messages"])
+ assert _replayed_prefix(later, count) == _replayed_prefix(earlier, count)
+
+
+@pytest.mark.parametrize("reminder_after_user", [True, False])
+def test_chat_flagged_model_replays_a_byte_identical_prefix_around_a_mid_conversation_reminder(
+ local_model_cost_map, reminder_after_user
+):
+ """Preserved thinking binds each signed block to the request prefix it was created
+ under (``system``, ``tools`` and the earlier messages), so turn N's transformed
+ request must be a byte-identical prefix of turn N+1's or the block is dropped."""
+ requests = [
+ VertexAIAnthropicConfig().transform_request(
+ model="claude-fable-5-1", messages=copy.deepcopy(turn), optional_params={}, litellm_params={}, headers={}
+ )
+ for turn in _preserved_thinking_turns(reminder_after_user)
+ ]
+
+ _assert_prefix_stable(requests)
+ assert [m["role"] for m in requests[1]["messages"]] == ["user", "assistant", "user", "system"]
+ assert [m["role"] for m in requests[2]["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
index 1b696669724..fd48688df84 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py
@@ -4,10 +4,15 @@ Tests PII detection and masking for different message formats
"""
import asyncio
+import copy
import json
+import re
from contextlib import asynccontextmanager
+from typing import Final
from unittest.mock import MagicMock, patch
+from aiohttp import web
+from aiohttp.test_utils import TestServer
import pytest
@@ -3849,3 +3854,79 @@ async def test_chunk_fanout_bound_is_shared_across_concurrent_calls():
)
assert state["peak"] >= 2
assert state["peak"] <= PRESIDIO_ANALYZE_CHUNK_CONCURRENCY
+
+
+_PERSON_NAME: Final = re.compile(r"\b[A-Z][a-z]+ [A-Z][a-z]+\b")
+
+
+def _person_spans(text: str) -> list[dict]:
+ return [
+ {"entity_type": "PERSON", "start": match.start(), "end": match.end(), "score": 0.85, "analysis_explanation": None}
+ for match in _PERSON_NAME.finditer(text)
+ ]
+
+
+def _redacted(text: str, spans: list[dict]) -> str:
+ starts = [0, *(span["end"] for span in spans)]
+ ends = [*(span["start"] for span in spans), len(text)]
+ return "".join(text[start:end] for start, end in zip(starts, ends))
+
+
+async def _fake_analyze(request: web.Request) -> web.Response:
+ payload = await request.json()
+ return web.json_response(_person_spans(payload["text"]))
+
+
+async def _fake_anonymize(request: web.Request) -> web.Response:
+ payload = await request.json()
+ spans = payload["analyzer_results"]
+ items = [{"entity_type": span["entity_type"], "operator": "replace"} for span in spans]
+ return web.json_response({"text": _redacted(payload["text"], spans), "items": items})
+
+
+def _fake_presidio_app() -> web.Application:
+ app = web.Application()
+ app.router.add_post("/analyze", _fake_analyze)
+ app.router.add_post("/anonymize", _fake_anonymize)
+ return app
+
+
+def _pii_turns() -> tuple[list[dict], list[dict]]:
+ turn_n = [
+ {"role": "system", "content": "You are terse."},
+ {"role": "user", "content": "My name is John Smith and my colleague is Alice Brown."},
+ ]
+ reply = {
+ "role": "assistant",
+ "content": "Noted.",
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": "sig-1"}],
+ }
+ return turn_n, [*turn_n, reply, {"role": "user", "content": "Now compare against Bob Jones too."}]
+
+
+async def test_pii_masking_replays_a_byte_identical_prefix_across_turns(mock_user_api_key, mock_cache):
+ """Masking rewrites the history on every turn, so the rewrite of an earlier message
+ must not depend on the turns that came after it or the signed thinking blocks in
+ the history lose their binding. The analyzer and anonymizer are an in-process fake
+ handed to the guardrail through its api_base settings."""
+ async with TestServer(_fake_presidio_app()) as server:
+ guardrail = _OPTIONAL_PresidioPIIMasking(
+ presidio_analyzer_api_base=str(server.make_url("/")),
+ presidio_anonymizer_api_base=str(server.make_url("/")),
+ pii_entities_config={PiiEntityType.PERSON: PiiAction.MASK},
+ )
+ masked = [
+ await guardrail.async_pre_call_hook(
+ user_api_key_dict=mock_user_api_key,
+ cache=mock_cache,
+ data={"model": "claude-fable-5-1", "messages": copy.deepcopy(turn)},
+ call_type="completion",
+ )
+ for turn in _pii_turns()
+ ]
+ await guardrail._close_http_session()
+ earlier, later = (result["messages"] for result in masked)
+
+ assert json.dumps(later[: len(earlier)], sort_keys=True) == json.dumps(earlier, sort_keys=True)
+ assert earlier[1]["content"] == "My name is and my colleague is ."
+ assert later[3]["content"] == "Now compare against too."
diff --git a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
index 3846a94c9fe..5d97beeb3fc 100644
--- a/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
+++ b/tests/unit/llms/bedrock/chat/invoke_transformations/test_bedrock_chat_invoke_transformations_anthropic_claude3_transformation.py
@@ -1,5 +1,6 @@
import asyncio
import base64
+import copy
import json
import uuid
from types import SimpleNamespace
@@ -1010,3 +1011,102 @@ def test_bedrock_chat_invoke_eager_input_streaming_beta_not_duplicated_with_clie
)
assert result["anthropic_beta"] == [FINE_GRAINED_TOOL_STREAMING_BETA]
+
+
+def _mid_conversation_system_conversation() -> list[dict]:
+ return [
+ {"role": "system", "content": [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]},
+ {"role": "user", "content": "First question"},
+ {"role": "assistant", "content": "First answer"},
+ {"role": "user", "content": "Second question"},
+ {"role": "system", "content": "Answer with exactly one word."},
+ {"role": "assistant", "content": "Second answer"},
+ {"role": "user", "content": "Third question"},
+ ]
+
+
+def test_chat_unflagged_model_converts_mid_conversation_system_instead_of_hoisting(local_model_cost_map):
+ """A hoisted reminder rewrites the top-level system block and invalidates the
+ prompt cache for the whole conversation (#36559)."""
+ result = AmazonAnthropicClaudeConfig().transform_request(
+ model="invoke/us.anthropic.claude-opus-4-7",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "assistant", "user"]
+ texts = [b["text"] for b in result["messages"][2]["content"] if b.get("type") == "text"]
+ assert texts[0] == "Second question"
+ assert texts[-1] == "Answer with exactly one word."
+
+
+def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_model_cost_map):
+ result = AmazonAnthropicClaudeConfig().transform_request(
+ model="invoke/us.anthropic.claude-opus-4-8",
+ messages=_mid_conversation_system_conversation(),
+ optional_params={},
+ litellm_params={},
+ headers={},
+ )
+
+ assert result["system"] == [{"type": "text", "text": "You are terse.", "cache_control": {"type": "ephemeral"}}]
+ assert [m["role"] for m in result["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]
+ assert result["messages"][3] == {
+ "role": "system",
+ "content": [{"type": "text", "text": "Answer with exactly one word."}],
+ }
+
+
+def _thinking_reply(text: str) -> dict:
+ return {
+ "role": "assistant",
+ "content": text,
+ "thinking_blocks": [{"type": "thinking", "thinking": "Working it out.", "signature": f"sig-{text}"}],
+ }
+
+
+def _preserved_thinking_turns(reminder_after_user: bool) -> tuple[list[dict], list[dict], list[dict]]:
+ turn_n = [{"role": "system", "content": "You are terse."}, {"role": "user", "content": "First question"}]
+ reminder = {"role": "system", "content": "Answer with exactly one word."}
+ second_question = {"role": "user", "content": "Second question"}
+ second_turn = [second_question, reminder] if reminder_after_user else [reminder, second_question]
+ turn_n_plus_one = [*turn_n, _thinking_reply("First answer"), *second_turn]
+ turn_n_plus_two = [*turn_n_plus_one, _thinking_reply("Second answer"), {"role": "user", "content": "Third question"}]
+ return turn_n, turn_n_plus_one, turn_n_plus_two
+
+
+def _replayed_prefix(request: dict, message_count: int) -> str:
+ replayed = {
+ "system": request.get("system"),
+ "tools": request.get("tools"),
+ "messages": request["messages"][:message_count],
+ }
+ return json.dumps(replayed, sort_keys=True)
+
+
+def _assert_prefix_stable(requests: list[dict]) -> None:
+ for earlier, later in zip(requests, requests[1:]):
+ count = len(earlier["messages"])
+ assert _replayed_prefix(later, count) == _replayed_prefix(earlier, count)
+
+
+@pytest.mark.parametrize("reminder_after_user", [True, False])
+def test_chat_flagged_model_replays_a_byte_identical_prefix_around_a_mid_conversation_reminder(
+ local_model_cost_map, reminder_after_user
+):
+ """Preserved thinking binds each signed block to the request prefix it was created
+ under (``system``, ``tools`` and the earlier messages), so turn N's transformed
+ request must be a byte-identical prefix of turn N+1's or the block is dropped."""
+ requests = [
+ AmazonAnthropicClaudeConfig().transform_request(
+ model="invoke/us.anthropic.claude-fable-5-1", messages=copy.deepcopy(turn), optional_params={}, litellm_params={}, headers={}
+ )
+ for turn in _preserved_thinking_turns(reminder_after_user)
+ ]
+
+ _assert_prefix_stable(requests)
+ assert [m["role"] for m in requests[1]["messages"]] == ["user", "assistant", "user", "system"]
+ assert [m["role"] for m in requests[2]["messages"]] == ["user", "assistant", "user", "system", "assistant", "user"]