scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-13 23:11:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 101

Merge pull request #32872 from BerriAI/litellm_cherrypick_1_91_x

Browse source 
fix(bedrock): backport #32578 and #32831 to stable/1.91.x for v1.91.2
This commit is contained in:
Mateo Wang 2026-07-10 22:56:58 -07:00 • committed by GitHub
commit 6950a52a15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 215 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py
View file

@ -93,26 +93,48 @@ class AmazonAnthropicClaudeMessagesConfig(
return [{"type": "text", "text": value}]
return [value]
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict) -> None:
"""Bedrock Invoke rejects ``role: "system"`` entries inside ``messages`` on
some Claude aliases; Anthropic Messages carries that content in the
top-level ``system`` field. Move any such entries into ``system`` before
the Invoke request is built."""
@staticmethod
def _is_system_role_message(message: Any) -> bool:
return isinstance(message, dict) and message.get("role") == "system"
def _normalize_system_role_messages_for_bedrock(self, anthropic_messages_request: dict, model: str) -> None:
"""Bedrock Invoke validates ``role: "system"`` entries inside ``messages``
per model. Models carrying ``supports_mid_conversation_system`` in the
cost map (the Opus 4.8 family) only reject a leading run ("messages.0:
use the top-level 'system' parameter for the initial system prompt") and
accept mid-conversation entries (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) in place, where they
MUST stay: hoisting one mutates the ``system`` prefix and invalidates the
prompt cache for the entire message history. Older Claude models (Opus
4.7, Sonnet 4.6, Haiku 4.5, ...) reject the role in every position
("role 'system' is not supported on this model"), so without the flag
every system entry is hoisted into the top-level ``system`` field.
Billing-header system blocks are stripped from the top-level ``system``
field regardless of whether anything was hoisted."""
messages = anthropic_messages_request.get("messages")
if not isinstance(messages, list):
return
system_role_messages = [m for m in messages if isinstance(m, dict) and m.get("role") == "system"]
if not system_role_messages:
return
anthropic_messages_request["messages"] = [
m for m in messages if not (isinstance(m, dict) and m.get("role") == "system")
]
if _supports_factory(
model=model,
custom_llm_provider="bedrock",
key="supports_mid_conversation_system",
):
leading_count = next(
(i for i, m in enumerate(messages) if not self._is_system_role_message(m)),
len(messages),
)
hoisted = messages[:leading_count]
remaining = messages[leading_count:]
else:
hoisted = [m for m in messages if self._is_system_role_message(m)]
remaining = [m for m in messages if not self._is_system_role_message(m)]
if hoisted:
anthropic_messages_request["messages"] = remaining
system_content = [
block
for source in (
anthropic_messages_request.get("system"),
*(m.get("content") for m in system_role_messages),
*(m.get("content") for m in hoisted),
)
for block in self._as_system_content_blocks(source)
]
@ -648,7 +670,7 @@ class AmazonAnthropicClaudeMessagesConfig(
litellm_params=litellm_params,
headers=headers,
)
self._normalize_system_role_messages_for_bedrock(anthropic_messages_request)
self._normalize_system_role_messages_for_bedrock(anthropic_messages_request, model=model)
#########################################################
############## BEDROCK Invoke SPECIFIC TRANSFORMATION ###
#########################################################

9
litellm/model_prices_and_context_window_backup.json
View file

@ -1360,6 +1360,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1394,6 +1395,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1428,6 +1430,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1462,6 +1465,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1481,6 +1485,7 @@
"anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1516,6 +1521,7 @@
"global.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1551,6 +1557,7 @@
"us.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1586,6 +1593,7 @@
"eu.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1621,6 +1629,7 @@
"au.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,

1
litellm/types/utils.py
View file

@ -143,6 +143,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
supports_web_search: Optional[bool]
supports_reasoning: Optional[bool]
supports_adaptive_thinking: Optional[bool]
supports_mid_conversation_system: Optional[bool]
supports_url_context: Optional[bool]
supports_none_reasoning_effort: Optional[bool]
supports_minimal_reasoning_effort: Optional[bool]

1
litellm/utils.py
View file

@ -5412,6 +5412,7 @@ def _get_model_info_helper(
supports_url_context=_model_info.get("supports_url_context", None),
supports_reasoning=_model_info.get("supports_reasoning", None),
supports_adaptive_thinking=_model_info.get("supports_adaptive_thinking", None),
supports_mid_conversation_system=_model_info.get("supports_mid_conversation_system", None),
supports_none_reasoning_effort=_model_info.get("supports_none_reasoning_effort", None),
supports_minimal_reasoning_effort=_model_info.get("supports_minimal_reasoning_effort", None),
supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),

9
model_prices_and_context_window.json
View file

@ -1360,6 +1360,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1394,6 +1395,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1428,6 +1430,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1462,6 +1465,7 @@
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
@ -1481,6 +1485,7 @@
"anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1516,6 +1521,7 @@
"global.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.25e-06,
"cache_creation_input_token_cost_above_1hr": 1e-05,
"cache_read_input_token_cost": 5e-07,
@ -1551,6 +1557,7 @@
"us.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1586,6 +1593,7 @@
"eu.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,
@ -1621,6 +1629,7 @@
"au.anthropic.claude-opus-4-8": {
"supports_parallel_tool_use_config": true,
"bedrock_converse_supports_strict_tools": false,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 6.875e-06,
"cache_creation_input_token_cost_above_1hr": 1.1e-05,
"cache_read_input_token_cost": 5.5e-07,

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm"
version = "1.91.1"
version = "1.91.2"
description = "Library to easily interface with LLM API providers"
readme = "README.md"
requires-python = ">=3.10, <3.14"
@ -269,7 +269,7 @@ source-exclude = [
profile = "black"
[tool.commitizen]
version = "1.91.1"
version = "1.91.2"
version_files = [
"pyproject.toml:^version",
]

7
tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py
View file

@ -527,12 +527,11 @@ def test_backward_compatibility_regular_nova_model():
assert result["imageGenerationConfig"]["cfg_scale"] == 7
def test_amazon_titan_image_gen():
"""Test Amazon Titan image generation with cost tracking."""
def test_amazon_nova_canvas_image_gen():
"""Test Amazon Nova Canvas image generation with cost tracking."""
from litellm import image_generation
# Use v2 as v1 has reached end of life
model_id = "bedrock/amazon.titan-image-generator-v2:0"
model_id = "bedrock/amazon.nova-canvas-v1:0"
response = litellm.image_generation(
model=model_id,

152
tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py
View file

@ -1893,6 +1893,158 @@ def test_bedrock_invoke_transform_merges_list_content_system_role_into_system():
]
@pytest.mark.parametrize(
"model",
[
"anthropic.claude-opus-4-8",
"us.anthropic.claude-fable-5",
],
)
def test_bedrock_invoke_transform_keeps_mid_conversation_system_role_in_place(local_model_cost_map, model):
"""Regression test for the Bedrock prompt-cache collapse: hoisting a
mid-conversation ``role: "system"`` message (e.g. Claude Code's
``mid-conversation-system-2026-04-07`` reminders) into the top-level
``system`` field mutates the cache prefix and invalidates the cached message
history, so on models flagged ``supports_mid_conversation_system`` (Claude
4.8+, which Invoke accepts the role on) such entries must be forwarded
in place. Billing-header blocks must still be stripped from the top-level
``system`` field even when nothing is hoisted."""
from litellm.types.router import GenericLiteLLMParams
cfg = AmazonAnthropicClaudeMessagesConfig()
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = cfg.transform_anthropic_messages_request(
model=model,
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params={
"max_tokens": 256,
"stream": False,
"system": [
{"type": "text", "text": "x-anthropic-billing-header: cc_version=2.1.205;"},
{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}},
],
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert result["messages"] == messages
assert result["system"] == [
{"type": "text", "text": "Base.", "cache_control": {"type": "ephemeral"}}
]
def test_bedrock_invoke_transform_hoists_only_leading_system_run(local_model_cost_map):
"""On models flagged ``supports_mid_conversation_system``, only the leading
run of ``role: "system"`` messages is hoisted into the top-level ``system``
field; a later system entry keeps its position in ``messages`` so the
serialized prefix stays stable across turns."""
from litellm.types.router import GenericLiteLLMParams
cfg = AmazonAnthropicClaudeMessagesConfig()
messages = [
{"role": "system", "content": "You are terse."},
{"role": "system", "content": "Cite sources."},
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
result = cfg.transform_anthropic_messages_request(
model="anthropic.claude-opus-4-8",
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert result["messages"] == [
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "You are terse."},
{"type": "text", "text": "Cite sources."},
]
def test_bedrock_invoke_transform_hoists_mid_conversation_system_for_older_claude(local_model_cost_map):
"""Regression test for Claude Code 400s on pre-Opus-4.8 Bedrock models:
Invoke rejects ``role: "system"`` in every position on Opus 4.7, Sonnet 4.6,
Haiku 4.5, etc. ("role 'system' is not supported on this model"), so on
models without ``supports_mid_conversation_system`` every system entry must
be hoisted into the top-level ``system`` field, mid-conversation ones
included."""
from litellm.types.router import GenericLiteLLMParams
cfg = AmazonAnthropicClaudeMessagesConfig()
messages = [
{"role": "user", "content": "read the file"},
{"role": "system", "content": "[Truncated: PARTIAL view of big1.txt]"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
result = cfg.transform_anthropic_messages_request(
model="us.anthropic.claude-opus-4-7",
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params={
"max_tokens": 256,
"stream": False,
"system": [{"type": "text", "text": "Base."}],
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert result["messages"] == [
{"role": "user", "content": "read the file"},
{"role": "assistant", "content": "reading"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [
{"type": "text", "text": "Base."},
{"type": "text", "text": "[Truncated: PARTIAL view of big1.txt]"},
]
def test_bedrock_invoke_transform_hoists_all_system_for_unmapped_model(local_model_cost_map):
"""A model with no cost-map entry and no fallback-generalization rule gets
the hoist-everything behavior: the safe default is a mutated cache prefix,
never a provider 400 from forwarding a role the model may not accept."""
from litellm.types.router import GenericLiteLLMParams
cfg = AmazonAnthropicClaudeMessagesConfig()
messages = [
{"role": "user", "content": "hi"},
{"role": "system", "content": "mid-conversation reminder"},
{"role": "assistant", "content": "hello"},
{"role": "user", "content": "continue"},
]
result = cfg.transform_anthropic_messages_request(
model="us.anthropic.claude-opus-3-9",
messages=copy.deepcopy(messages),
anthropic_messages_optional_request_params={"max_tokens": 256, "stream": False},
litellm_params=GenericLiteLLMParams(),
headers={},
)
assert result["messages"] == [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "hello"},
{"role": "user", "content": "continue"},
]
assert result["system"] == [{"type": "text", "text": "mid-conversation reminder"}]
def test_as_system_content_blocks_handles_each_shape():
"""``_as_system_content_blocks`` normalizes every system shape: ``None`` -> empty,
a string -> a single text block, a list -> a shallow copy, and any other value

1
tests/test_litellm/test_utils.py
View file

@ -836,6 +836,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"supports_xhigh_reasoning_effort": {"type": "boolean"},
"supports_max_reasoning_effort": {"type": "boolean"},
"supports_adaptive_thinking": {"type": "boolean"},
"supports_mid_conversation_system": {"type": "boolean"},
"supports_sampling_params": {"type": "boolean"},
"supports_output_config": {"type": "boolean"},
"supports_speed": {"type": "boolean"},

2
uv.lock generated
View file

@ -3232,7 +3232,7 @@ wheels = [
[[package]]
name = "litellm"
version = "1.91.1"
version = "1.91.2"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },