From 951e0ae29b6f3afcee8812ec8cd639ec79cbc41b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 10 Jul 2026 20:38:52 -0700 Subject: [PATCH] fix(bedrock): gate in-place system role messages on model support for Claude Invoke (#32831) Backport of #32831 to stable/1.91.x. Cherry-picked from 5e23a5ab053f56a88cb41a3bc468ae9f61841891 (litellm_internal_staging). Adapted for this line: the fallback-generalizations feature does not exist on stable/1.91.x, so the fallback rule for unmapped Claude 4.8+ models (and its tests) is omitted; unmapped models fall back to hoist-all, which is the safe default. Only supports_mid_conversation_system is added to the Opus 4.8 cost map entries. --- .../anthropic_claude3_transformation.py | 51 +++++++---- ...odel_prices_and_context_window_backup.json | 5 ++ litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 5 ++ .../test_anthropic_claude3_transformation.py | 88 +++++++++++++++++-- tests/test_litellm/test_utils.py | 1 + 7 files changed, 127 insertions(+), 25 deletions(-) diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index d44b1f4cccb..88e2901a372 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -93,31 +93,48 @@ class AmazonAnthropicClaudeMessagesConfig( return [{"type": "text", "text": value}] return [value] - def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None: - """Bedrock Invoke rejects a conversation that opens with ``role: "system"`` - entries inside ``messages`` ("messages.0: use the top-level 'system' - parameter for the initial system prompt"); Anthropic Messages carries that - content in the top-level ``system`` field, so hoist the leading run of - system entries there. Mid-conversation system entries (e.g. Claude Code's - ``mid-conversation-system-2026-04-07`` reminders) are accepted by Invoke in - place and MUST stay in place: hoisting one mutates the ``system`` prefix - and invalidates the prompt cache for the entire message history. + @staticmethod + def _is_system_role_message(message: Any) -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None: + """Bedrock Invoke validates ``role: "system"`` entries inside ``messages`` + per model. Models carrying ``supports_mid_conversation_system`` in the + cost map (the Opus 4.8 family) only reject a leading run ("messages.0: + use the top-level 'system' parameter for the initial system prompt") and + accept mid-conversation entries (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) in place, where they + MUST stay: hoisting one mutates the ``system`` prefix and invalidates the + prompt cache for the entire message history. Older Claude models (Opus + 4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position + ("role 'system' is not supported on this model"), so without the flag + every system entry is hoisted into the top-level ``system`` field. Billing-header system blocks are stripped from the top-level ``system`` field regardless of whether anything was hoisted.""" messages = anthropic_messages_request.get("messages") if not isinstance(messages, list): return - leading_count = next( - (i for i, m in enumerate(messages) if not (isinstance(m, dict) and m.get("role") == "system")), - len(messages), - ) - if leading_count: - anthropic_messages_request["messages"] = messages[leading_count:] + if _supports_factory( + model=model, + custom_llm_provider="bedrock", + key="supports_mid_conversation_system", + ): + leading_count = next( + (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + len(messages), + ) + hoisted = messages[:leading_count] + remaining = messages[leading_count:] + else: + hoisted = [m for m in messages if self._is_system_role_message(m)] + remaining = [m for m in messages if not self._is_system_role_message(m)] + if hoisted: + anthropic_messages_request["messages"] = remaining system_content = [ block for source in ( anthropic_messages_request.get("system"), - *(m.get("content") for m in messages[:leading_count]), + *(m.get("content") for m in hoisted), ) for block in self._as_system_content_blocks(source) ] @@ -653,7 +670,7 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages_for_bedrock(anthropic_messages_request) + self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fef4301017a..c9b8a52bbb7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1481,6 +1481,7 @@ "anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1517,7 @@ "global.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1553,7 @@ "us.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1589,7 @@ "eu.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1625,7 @@ "au.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index feba5652f10..c277beb4589 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -143,6 +143,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_web_search: Optional[bool] supports_reasoning: Optional[bool] supports_adaptive_thinking: Optional[bool] + supports_mid_conversation_system: Optional[bool] supports_url_context: Optional[bool] supports_none_reasoning_effort: Optional[bool] supports_minimal_reasoning_effort: Optional[bool] diff --git a/litellm/utils.py b/litellm/utils.py index e876f857c68..07a055e4fca 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5412,6 +5412,7 @@ def _get_model_info_helper( supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), + supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15fd1052d91..ec052070e49 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1481,6 +1481,7 @@ "anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1517,7 @@ "global.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1553,7 @@ "us.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1589,7 @@ "eu.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1625,7 @@ "au.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index adf2f4c972f..b0425f90fae 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1893,14 +1893,15 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system(): ] -def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(): +def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map): """Regression test for the Bedrock prompt-cache collapse: hoisting a mid-conversation ``role: "system"`` message (e.g. Claude Code's ``mid-conversation-system-2026-04-07`` reminders) into the top-level ``system`` field mutates the cache prefix and invalidates the cached message - history, so such entries must be forwarded in place. Invoke only rejects a - system entry at ``messages.0``. Billing-header blocks must still be stripped - from the top-level ``system`` field even when nothing is hoisted.""" + history, so on models flagged ``supports_mid_conversation_system`` (the Opus + 4.8 family, which Invoke accepts the role on) such entries must be forwarded + in place. Billing-header blocks must still be stripped from the top-level + ``system`` field even when nothing is hoisted.""" from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() @@ -1932,10 +1933,11 @@ def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(): ] -def test_bedrock_invoke_transform_hoists_only_leading_system_run(): - """Only the leading run of ``role: "system"`` messages is hoisted into the - top-level ``system`` field; a later system entry keeps its position in - ``messages`` so the serialized prefix stays stable across turns.""" +def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): + """On models flagged ``supports_mid_conversation_system``, only the leading + run of ``role: "system"`` messages is hoisted into the top-level ``system`` + field; a later system entry keeps its position in ``messages`` so the + serialized prefix stays stable across turns.""" from litellm.types.router import GenericLiteLLMParams cfg = AmazonAnthropicClaudeMessagesConfig() @@ -1966,6 +1968,76 @@ def test_bedrock_invoke_transform_hoists_only_leading_system_run(): ] +def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claude(local_model_cost_map): + """Regression test for Claude Code 400s on pre-Opus-4.8 Bedrock models: + Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, + Haiku 4.5, etc. ("role 'system' is not supported on this model"), so on + models without ``supports_mid_conversation_system`` every system entry must + be hoisted into the top-level ``system`` field, mid-conversation ones + included.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-4-7", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [{"type": "text", "text": "Base."}], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "read the file"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "Base."}, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ] + + +def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_model_cost_map): + """A model with no cost-map entry and no fallback-generalization rule gets + the hoist-everything behavior: the safe default is a mutated cache prefix, + never a provider 400 from forwarding a role the model may not accept.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-3-9", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [{"type": "text", "text": "mid-conversation reminder"}] + + def test_as_system_content_blocks_handles_each_shape(): """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 9ceb94ef9bf..ad7f58c92d1 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -836,6 +836,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"},