mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
fix(bedrock): keep mid-conversation system messages in place for Claude Invoke (#32578)
Backport of #32578 to stable/1.91.x.
Cherry-picked from cc36d5469c (litellm_internal_staging).
This commit is contained in:
parent
cdc8c72c97
commit
62f239b09b
2 changed files with 90 additions and 12 deletions
|
|
@ -94,25 +94,30 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
return [value]
|
||||
|
||||
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None:
|
||||
"""Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on
|
||||
some Claude aliases; Anthropic Messages carries that content in the
|
||||
top-level ``system`` field. Move any such entries into ``system`` before
|
||||
the Invoke request is built."""
|
||||
"""Bedrock Invoke rejects a conversation that opens with ``role: "system"``
|
||||
entries inside ``messages`` ("messages.0: use the top-level 'system'
|
||||
parameter for the initial system prompt"); Anthropic Messages carries that
|
||||
content in the top-level ``system`` field, so hoist the leading run of
|
||||
system entries there. Mid-conversation system entries (e.g. Claude Code's
|
||||
``mid-conversation-system-2026-04-07`` reminders) are accepted by Invoke in
|
||||
place and MUST stay in place: hoisting one mutates the ``system`` prefix
|
||||
and invalidates the prompt cache for the entire message history.
|
||||
Billing-header system blocks are stripped from the top-level ``system``
|
||||
field regardless of whether anything was hoisted."""
|
||||
messages = anthropic_messages_request.get("messages")
|
||||
if not isinstance(messages, list):
|
||||
return
|
||||
system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"]
|
||||
if not system_role_messages:
|
||||
return
|
||||
|
||||
anthropic_messages_request["messages"] = [
|
||||
m for m in messages if not (isinstance(m, dict) and m.get("role") == "system")
|
||||
]
|
||||
leading_count = next(
|
||||
(i for i, m in enumerate(messages) if not (isinstance(m, dict) and m.get("role") == "system")),
|
||||
len(messages),
|
||||
)
|
||||
if leading_count:
|
||||
anthropic_messages_request["messages"] = messages[leading_count:]
|
||||
system_content = [
|
||||
block
|
||||
for source in (
|
||||
anthropic_messages_request.get("system"),
|
||||
*(m.get("content") for m in system_role_messages),
|
||||
*(m.get("content") for m in messages[:leading_count]),
|
||||
)
|
||||
for block in self._as_system_content_blocks(source)
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1893,6 +1893,79 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system():
|
|||
]
|
||||
|
||||
|
||||
def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place():
|
||||
"""Regression test for the Bedrock prompt-cache collapse: hoisting a
|
||||
mid-conversation ``role: "system"`` message (e.g. Claude Code's
|
||||
``mid-conversation-system-2026-04-07`` reminders) into the top-level
|
||||
``system`` field mutates the cache prefix and invalidates the cached message
|
||||
history, so such entries must be forwarded in place. Invoke only rejects a
|
||||
system entry at ``messages.0``. Billing-header blocks must still be stripped
|
||||
from the top-level ``system`` field even when nothing is hoisted."""
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
messages = [
|
||||
{"role": "user", "content": "read the file"},
|
||||
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
|
||||
{"role": "assistant", "content": "reading"},
|
||||
{"role": "user", "content": "continue"},
|
||||
]
|
||||
|
||||
result = cfg.transform_anthropic_messages_request(
|
||||
model="anthropic.claude-opus-4-8",
|
||||
messages=copy.deepcopy(messages),
|
||||
anthropic_messages_optional_request_params={
|
||||
"max_tokens": 256,
|
||||
"stream": False,
|
||||
"system": [
|
||||
{"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.205;"},
|
||||
{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["messages"] == messages
|
||||
assert result["system"] == [
|
||||
{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}
|
||||
]
|
||||
|
||||
|
||||
def test_bedrock_invoke_transform_hoists_only_leading_system_run():
|
||||
"""Only the leading run of ``role: "system"`` messages is hoisted into the
|
||||
top-level ``system`` field; a later system entry keeps its position in
|
||||
``messages`` so the serialized prefix stays stable across turns."""
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
cfg = AmazonAnthropicClaudeMessagesConfig()
|
||||
messages = [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{"role": "system", "content": "Cite sources."},
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "system", "content": "mid-conversation reminder"},
|
||||
{"role": "user", "content": "continue"},
|
||||
]
|
||||
|
||||
result = cfg.transform_anthropic_messages_request(
|
||||
model="anthropic.claude-opus-4-8",
|
||||
messages=copy.deepcopy(messages),
|
||||
anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False},
|
||||
litellm_params=GenericLiteLLMParams(),
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert result["messages"] == [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "system", "content": "mid-conversation reminder"},
|
||||
{"role": "user", "content": "continue"},
|
||||
]
|
||||
assert result["system"] == [
|
||||
{"type": "text", "text": "You are terse."},
|
||||
{"type": "text", "text": "Cite sources."},
|
||||
]
|
||||
|
||||
|
||||
def test_as_system_content_blocks_handles_each_shape():
|
||||
"""``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
|
||||
a string -> a single text block, a list -> a shallow copy, and any other value
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue