diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index f5309d521a9..88e2901a372 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -93,26 +93,48 @@ class AmazonAnthropicClaudeMessagesConfig( return [{"type": "text", "text": value}] return [value] - def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None: - """Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on - some Claude aliases; Anthropic Messages carries that content in the - top-level ``system`` field. Move any such entries into ``system`` before - the Invoke request is built.""" + @staticmethod + def _is_system_role_message(message: Any) -> bool: + return isinstance(message, dict) and message.get("role") == "system" + + def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None: + """Bedrock Invoke validates ``role: "system"`` entries inside ``messages`` + per model. Models carrying ``supports_mid_conversation_system`` in the + cost map (the Opus 4.8 family) only reject a leading run ("messages.0: + use the top-level 'system' parameter for the initial system prompt") and + accept mid-conversation entries (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) in place, where they + MUST stay: hoisting one mutates the ``system`` prefix and invalidates the + prompt cache for the entire message history. Older Claude models (Opus + 4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position + ("role 'system' is not supported on this model"), so without the flag + every system entry is hoisted into the top-level ``system`` field. + Billing-header system blocks are stripped from the top-level ``system`` + field regardless of whether anything was hoisted.""" messages = anthropic_messages_request.get("messages") if not isinstance(messages, list): return - system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"] - if not system_role_messages: - return - - anthropic_messages_request["messages"] = [ - m for m in messages if not (isinstance(m, dict) and m.get("role") == "system") - ] + if _supports_factory( + model=model, + custom_llm_provider="bedrock", + key="supports_mid_conversation_system", + ): + leading_count = next( + (i for i, m in enumerate(messages) if not self._is_system_role_message(m)), + len(messages), + ) + hoisted = messages[:leading_count] + remaining = messages[leading_count:] + else: + hoisted = [m for m in messages if self._is_system_role_message(m)] + remaining = [m for m in messages if not self._is_system_role_message(m)] + if hoisted: + anthropic_messages_request["messages"] = remaining system_content = [ block for source in ( anthropic_messages_request.get("system"), - *(m.get("content") for m in system_role_messages), + *(m.get("content") for m in hoisted), ) for block in self._as_system_content_blocks(source) ] @@ -648,7 +670,7 @@ class AmazonAnthropicClaudeMessagesConfig( litellm_params=litellm_params, headers=headers, ) - self._normalize_system_role_messages_for_bedrock(anthropic_messages_request) + self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model) ######################################################### ############## BEDROCK Invoke SPECIFIC TRANSFORMATION ### ######################################################### diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fef4301017a..1ec59d8b3f8 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -1360,6 +1360,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1394,6 +1395,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1428,6 +1430,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1462,6 +1465,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1481,6 +1485,7 @@ "anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1521,7 @@ "global.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1557,7 @@ "us.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1593,7 @@ "eu.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1629,7 @@ "au.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index feba5652f10..c277beb4589 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -143,6 +143,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_web_search: Optional[bool] supports_reasoning: Optional[bool] supports_adaptive_thinking: Optional[bool] + supports_mid_conversation_system: Optional[bool] supports_url_context: Optional[bool] supports_none_reasoning_effort: Optional[bool] supports_minimal_reasoning_effort: Optional[bool] diff --git a/litellm/utils.py b/litellm/utils.py index e876f857c68..07a055e4fca 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5412,6 +5412,7 @@ def _get_model_info_helper( supports_url_context=_model_info.get("supports_url_context", None), supports_reasoning=_model_info.get("supports_reasoning", None), supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None), + supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None), supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None), supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None), supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15fd1052d91..efc9ab5a817 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -1360,6 +1360,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1394,6 +1395,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1428,6 +1430,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1462,6 +1465,7 @@ "search_context_size_medium": 0.01 }, "supports_adaptive_thinking": true, + "supports_mid_conversation_system": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1481,6 +1485,7 @@ "anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1516,6 +1521,7 @@ "global.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, @@ -1551,6 +1557,7 @@ "us.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1586,6 +1593,7 @@ "eu.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, @@ -1621,6 +1629,7 @@ "au.anthropic.claude-opus-4-8": { "supports_parallel_tool_use_config": true, "bedrock_converse_supports_strict_tools": false, + "supports_mid_conversation_system": true, "cache_creation_input_token_cost": 6.875e-06, "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, diff --git a/pyproject.toml b/pyproject.toml index 09c6ed70d3d..33f500401ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.91.1" +version = "1.91.2" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.14" @@ -269,7 +269,7 @@ source-exclude = [ profile = "black" [tool.commitizen] -version = "1.91.1" +version = "1.91.2" version_files = [ "pyproject.toml:^version", ] diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py index 36ae9e1df67..6925bb2abc5 100644 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py @@ -527,12 +527,11 @@ def test_backward_compatibility_regular_nova_model(): assert result["imageGenerationConfig"]["cfg_scale"] == 7 -def test_amazon_titan_image_gen(): - """Test Amazon Titan image generation with cost tracking.""" +def test_amazon_nova_canvas_image_gen(): + """Test Amazon Nova Canvas image generation with cost tracking.""" from litellm import image_generation - # Use v2 as v1 has reached end of life - model_id = "bedrock/amazon.titan-image-generator-v2:0" + model_id = "bedrock/amazon.nova-canvas-v1:0" response = litellm.image_generation( model=model_id, diff --git a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py index 03d0d87a58c..a56a1422a35 100644 --- a/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py +++ b/tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py @@ -1893,6 +1893,158 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system(): ] +@pytest.mark.parametrize( + "model", + [ + "anthropic.claude-opus-4-8", + "us.anthropic.claude-fable-5", + ], +) +def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map, model): + """Regression test for the Bedrock prompt-cache collapse: hoisting a + mid-conversation ``role: "system"`` message (e.g. Claude Code's + ``mid-conversation-system-2026-04-07`` reminders) into the top-level + ``system`` field mutates the cache prefix and invalidates the cached message + history, so on models flagged ``supports_mid_conversation_system`` (Claude + 4.8+, which Invoke accepts the role on) such entries must be forwarded + in place. Billing-header blocks must still be stripped from the top-level + ``system`` field even when nothing is hoisted.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model=model, + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.205;"}, + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}, + ], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == messages + assert result["system"] == [ + {"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}} + ] + + +def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map): + """On models flagged ``supports_mid_conversation_system``, only the leading + run of ``role: "system"`` messages is hoisted into the top-level ``system`` + field; a later system entry keeps its position in ``messages`` so the + serialized prefix stays stable across turns.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "system", "content": "You are terse."}, + {"role": "system", "content": "Cite sources."}, + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="anthropic.claude-opus-4-8", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "You are terse."}, + {"type": "text", "text": "Cite sources."}, + ] + + +def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claude(local_model_cost_map): + """Regression test for Claude Code 400s on pre-Opus-4.8 Bedrock models: + Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6, + Haiku 4.5, etc. ("role 'system' is not supported on this model"), so on + models without ``supports_mid_conversation_system`` every system entry must + be hoisted into the top-level ``system`` field, mid-conversation ones + included.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "read the file"}, + {"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-4-7", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={ + "max_tokens": 256, + "stream": False, + "system": [{"type": "text", "text": "Base."}], + }, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "read the file"}, + {"role": "assistant", "content": "reading"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [ + {"type": "text", "text": "Base."}, + {"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"}, + ] + + +def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_model_cost_map): + """A model with no cost-map entry and no fallback-generalization rule gets + the hoist-everything behavior: the safe default is a mutated cache prefix, + never a provider 400 from forwarding a role the model may not accept.""" + from litellm.types.router import GenericLiteLLMParams + + cfg = AmazonAnthropicClaudeMessagesConfig() + messages = [ + {"role": "user", "content": "hi"}, + {"role": "system", "content": "mid-conversation reminder"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + + result = cfg.transform_anthropic_messages_request( + model="us.anthropic.claude-opus-3-9", + messages=copy.deepcopy(messages), + anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False}, + litellm_params=GenericLiteLLMParams(), + headers={}, + ) + + assert result["messages"] == [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + {"role": "user", "content": "continue"}, + ] + assert result["system"] == [{"type": "text", "text": "mid-conversation reminder"}] + + def test_as_system_content_blocks_handles_each_shape(): """``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty, a string -> a single text block, a list -> a shallow copy, and any other value diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py index 9ceb94ef9bf..ad7f58c92d1 100644 --- a/tests/test_litellm/test_utils.py +++ b/tests/test_litellm/test_utils.py @@ -836,6 +836,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "supports_xhigh_reasoning_effort": {"type": "boolean"}, "supports_max_reasoning_effort": {"type": "boolean"}, "supports_adaptive_thinking": {"type": "boolean"}, + "supports_mid_conversation_system": {"type": "boolean"}, "supports_sampling_params": {"type": "boolean"}, "supports_output_config": {"type": "boolean"}, "supports_speed": {"type": "boolean"}, diff --git a/uv.lock b/uv.lock index bf6de961edc..d2db4983ff8 100644 --- a/uv.lock +++ b/uv.lock @@ -3232,7 +3232,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.91.1" +version = "1.91.2" source = { editable = "." } dependencies = [ { name = "aiohttp" },