mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(bedrock): honor ttl for tool_config cache injection points (#31929)
Backport of #31929 to stable/1.90.x.
Cherry-picked from 1543725916 (litellm_internal_staging).
Deviations from the upstream squash, needed because the line predates some
staging-only state:
- model_prices JSON churn reduced to its semantic content: drop the stale
1h-TTL pricing keys from the two Claude 3.5 Sonnet Bedrock entries and add
supports_parallel_tool_use_config to the flagged entries. 6 of the 58
flagged entries (*.anthropic.claude-sonnet-5) do not exist on this line and
were skipped
- ported the local_model_cost_map fixture into
test_anthropic_claude3_transformation.py (added upstream by an earlier
staging commit that never reached this line)
- test_parallel_tool_calls_config_kept_for_sonnet_5 renamed and pointed at
anthropic.claude-sonnet-4-6, since sonnet-5 has no entry on this line
- test_parallel_tool_calls_newer_model_adds_disable_flag now forces the
bundled cost map so it does not depend on the remote map having synced the
new flag
- ruff-strict-budget.json kept at the line's schema (no change needed)
This commit is contained in:
Shivam Rawat
mateo-berri
parent
6a5353196d
commit
bb90565092
10 changed files with 683 additions and 229 deletions
|
|
@ -76,6 +76,7 @@ from litellm.utils import (
|
|||
from ..common_utils import (
|
||||
BedrockError,
|
||||
BedrockModelInfo,
|
||||
bedrock_converse_supports_parallel_tool_use_config,
|
||||
get_anthropic_beta_from_headers,
|
||||
get_bedrock_tool_name,
|
||||
is_claude_4_5_on_bedrock,
|
||||
|
|
@ -1176,18 +1177,34 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if cache_control is None:
|
||||
return None
|
||||
|
||||
cache_point = CachePointBlock(type="default")
|
||||
if isinstance(cache_control, dict) and "ttl" in cache_control:
|
||||
ttl = cache_control["ttl"]
|
||||
if ttl in ["5m", "1h"] and model is not None:
|
||||
if is_claude_4_5_on_bedrock(model):
|
||||
cache_point["ttl"] = ttl
|
||||
cache_point = self._build_cache_point_block(cache_control, model)
|
||||
|
||||
if block_type == "system":
|
||||
return SystemContentBlock(cachePoint=cache_point)
|
||||
else:
|
||||
return ContentBlock(cachePoint=cache_point)
|
||||
|
||||
@staticmethod
|
||||
def _build_cache_point_block(
|
||||
control: Optional[dict], model: Optional[str] = None
|
||||
) -> CachePointBlock:
|
||||
"""Build a Bedrock ``cachePoint`` block from an OpenAI-style ``cache_control``/``control`` dict.
|
||||
|
||||
``type`` is always ``"default"`` (the only value Bedrock's Converse API
|
||||
accepts). ``ttl`` is only honored for models that support extended TTL
|
||||
caching (Claude 4.5 family on Bedrock).
|
||||
"""
|
||||
cache_point = CachePointBlock(type="default")
|
||||
if isinstance(control, dict) and "ttl" in control:
|
||||
ttl = control["ttl"]
|
||||
if (
|
||||
ttl in ["5m", "1h"]
|
||||
and model is not None
|
||||
and is_claude_4_5_on_bedrock(model)
|
||||
):
|
||||
cache_point["ttl"] = ttl
|
||||
return cache_point
|
||||
|
||||
def _transform_system_message(
|
||||
self, messages: List[AllMessageValues], model: Optional[str] = None
|
||||
) -> Tuple[List[AllMessageValues], List[SystemContentBlock]]:
|
||||
|
|
@ -1339,7 +1356,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
parallel_tool_use_config = additional_request_params.pop(
|
||||
"_parallel_tool_use_config", None
|
||||
)
|
||||
if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model):
|
||||
if (
|
||||
parallel_tool_use_config is not None
|
||||
and bedrock_converse_supports_parallel_tool_use_config(model)
|
||||
):
|
||||
for key, value in parallel_tool_use_config.items():
|
||||
if (
|
||||
key in additional_request_params
|
||||
|
|
@ -1651,7 +1671,10 @@ class AmazonConverseConfig(BaseConfig):
|
|||
if cache_injection_points and len(bedrock_tools) > 0:
|
||||
for point in cache_injection_points:
|
||||
if point.get("location") == "tool_config":
|
||||
bedrock_tools.append({"cachePoint": {"type": "default"}})
|
||||
cache_point = self._build_cache_point_block(
|
||||
point.get("control"), model
|
||||
)
|
||||
bedrock_tools.append(ToolBlock(cachePoint=cache_point))
|
||||
break
|
||||
|
||||
bedrock_tool_config: Optional[ToolConfigBlock] = None
|
||||
|
|
|
|||
|
|
@ -652,39 +652,33 @@ def get_bedrock_base_model(model: str) -> str:
|
|||
return model
|
||||
|
||||
|
||||
def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool:
|
||||
return any(
|
||||
(litellm.model_cost.get(candidate) or {}).get(
|
||||
"supports_parallel_tool_use_config"
|
||||
)
|
||||
is True
|
||||
for candidate in (model, get_bedrock_base_model(model))
|
||||
)
|
||||
|
||||
|
||||
def is_claude_4_5_on_bedrock(model: str) -> bool:
|
||||
"""
|
||||
Check if the model is a Claude 4.5 model on Bedrock.
|
||||
Claude 4.5 models support prompt caching with '5m' and '1h' TTL on Bedrock.
|
||||
Check if the model supports Bedrock prompt caching with an extended '1h' TTL
|
||||
(in addition to the default 5m TTL).
|
||||
|
||||
Backed by the ``cache_creation_input_token_cost_above_1hr`` field in
|
||||
``model_prices_and_context_window.json`` instead of a hardcoded list of
|
||||
model-name patterns, so newly released models pick up support as soon as
|
||||
their pricing entry ships, with no code change required here.
|
||||
"""
|
||||
model_lower = model.lower()
|
||||
claude_4_5_patterns = [
|
||||
"sonnet-4.5",
|
||||
"sonnet_4.5",
|
||||
"sonnet-4-5",
|
||||
"sonnet_4_5",
|
||||
"haiku-4.5",
|
||||
"haiku_4.5",
|
||||
"haiku-4-5",
|
||||
"haiku_4_5",
|
||||
"opus-4.5",
|
||||
"opus_4.5",
|
||||
"opus-4-5",
|
||||
"opus_4_5",
|
||||
"sonnet-4.6",
|
||||
"sonnet_4.6",
|
||||
"sonnet-4-6",
|
||||
"sonnet_4_6",
|
||||
"opus-4.6",
|
||||
"opus_4.6",
|
||||
"opus-4-6",
|
||||
"opus_4_6",
|
||||
"opus-4.7",
|
||||
"opus_4.7",
|
||||
"opus-4-7",
|
||||
"opus_4_7",
|
||||
]
|
||||
return any(pattern in model_lower for pattern in claude_4_5_patterns)
|
||||
return any(
|
||||
(litellm.model_cost.get(candidate) or {}).get(
|
||||
"cache_creation_input_token_cost_above_1hr"
|
||||
)
|
||||
is not None
|
||||
for candidate in (model, get_bedrock_base_model(model))
|
||||
)
|
||||
|
||||
|
||||
_BEDROCK_MODEL_VERSION_SUFFIX_RE = re.compile(r"-v\d+(?::\d+)?$")
|
||||
|
|
|
|||
|
|
@ -734,7 +734,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-haiku-4-5@20251001": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
|
|
@ -758,7 +759,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -777,8 +779,6 @@
|
|||
"output_cost_per_token_above_200k_tokens": 3e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 7.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07
|
||||
},
|
||||
|
|
@ -803,9 +803,7 @@
|
|||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 3e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 7.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07
|
||||
},
|
||||
"anthropic.claude-3-7-sonnet-20240620-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -981,7 +979,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -1011,7 +1010,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -1041,7 +1041,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1071,7 +1072,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1101,7 +1103,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1131,7 +1134,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1164,7 +1168,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-mythos-preview": {
|
||||
"input_cost_per_token": 0,
|
||||
|
|
@ -1212,7 +1217,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1245,7 +1251,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1278,7 +1285,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1311,7 +1319,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
|
|
@ -1344,7 +1353,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
|
|
@ -1377,7 +1387,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.375e-05,
|
||||
|
|
@ -1410,7 +1421,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.375e-05,
|
||||
|
|
@ -1443,7 +1455,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1477,7 +1490,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1511,7 +1525,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1545,7 +1560,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1579,7 +1595,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1613,7 +1630,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1645,7 +1663,8 @@
|
|||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1674,7 +1693,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1703,7 +1723,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1732,7 +1753,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1761,7 +1783,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1790,7 +1813,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1819,7 +1843,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1880,7 +1905,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-v1": {
|
||||
"input_cost_per_token": 8e-06,
|
||||
|
|
@ -2130,7 +2156,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -2219,7 +2246,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"azure/ada": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -9360,7 +9388,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -9382,7 +9411,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
|
|
@ -9535,7 +9565,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -9557,7 +9588,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
|
|
@ -10287,7 +10319,8 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"claude-opus-4-1": {
|
||||
"cache_creation_input_token_cost": 1.875e-05,
|
||||
|
|
@ -14087,7 +14120,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -14300,7 +14334,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0": {
|
||||
"input_cost_per_token": 1.3e-07,
|
||||
|
|
@ -19826,7 +19861,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -19878,7 +19914,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.amazon.nova-2-lite-v1:0": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
|
|
@ -23987,7 +24024,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.375e-06,
|
||||
|
|
@ -24010,7 +24048,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"crusoe/deepseek-ai/DeepSeek-R1-0528": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -32555,7 +32594,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -32714,7 +32754,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -32741,7 +32782,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.375e-06,
|
||||
|
|
@ -32763,7 +32805,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.875e-05,
|
||||
|
|
@ -32817,7 +32860,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-5-20251101-v1:0": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -32846,7 +32890,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-5-20251101-v1:0": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -32874,7 +32919,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -42786,7 +42832,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true
|
||||
"supports_pdf_input": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.5e-06,
|
||||
|
|
@ -42809,7 +42856,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true
|
||||
"supports_pdf_input": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"soniox/stt-async-v4": {
|
||||
"litellm_provider": "soniox",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ class CacheControlToolConfigInjectionPoint(TypedDict):
|
|||
"""Type for tool_config-level injection points (Bedrock)."""
|
||||
|
||||
location: Literal["tool_config"]
|
||||
control: Optional[ChatCompletionCachedContent]
|
||||
|
||||
|
||||
CacheControlInjectionPoint = Union[
|
||||
|
|
|
|||
|
|
@ -734,7 +734,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-haiku-4-5@20251001": {
|
||||
"cache_creation_input_token_cost": 1.25e-06,
|
||||
|
|
@ -758,7 +759,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_streaming": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -777,8 +779,6 @@
|
|||
"output_cost_per_token_above_200k_tokens": 3e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 7.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05,
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
"cache_read_input_token_cost": 3e-07
|
||||
},
|
||||
|
|
@ -803,9 +803,7 @@
|
|||
"input_cost_per_token_above_200k_tokens": 6e-06,
|
||||
"output_cost_per_token_above_200k_tokens": 3e-05,
|
||||
"cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 7.5e-06,
|
||||
"cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.5e-05
|
||||
"cache_read_input_token_cost_above_200k_tokens": 6e-07
|
||||
},
|
||||
"anthropic.claude-3-7-sonnet-20240620-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -981,7 +979,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -1011,7 +1010,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -1041,7 +1041,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1071,7 +1072,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1101,7 +1103,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-6-v1": {
|
||||
"cache_creation_input_token_cost": 6.875e-06,
|
||||
|
|
@ -1131,7 +1134,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"bedrock_output_config_effort_ceiling": "max"
|
||||
"bedrock_output_config_effort_ceiling": "max",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1164,7 +1168,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-mythos-preview": {
|
||||
"input_cost_per_token": 0,
|
||||
|
|
@ -1212,7 +1217,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1245,7 +1251,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1278,7 +1285,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1311,7 +1319,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
|
|
@ -1344,7 +1353,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.25e-05,
|
||||
|
|
@ -1377,7 +1387,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.375e-05,
|
||||
|
|
@ -1410,7 +1421,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-fable-5": {
|
||||
"cache_creation_input_token_cost": 1.375e-05,
|
||||
|
|
@ -1443,7 +1455,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1477,7 +1490,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1511,7 +1525,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1545,7 +1560,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1579,7 +1595,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-opus-4-8": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1613,7 +1630,8 @@
|
|||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh"
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-opus-4-7": {
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
|
|
@ -1645,7 +1663,8 @@
|
|||
"tool_use_system_prompt_tokens": 346,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_minimal_reasoning_effort": true
|
||||
"supports_minimal_reasoning_effort": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1674,7 +1693,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1703,7 +1723,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1732,7 +1753,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1761,7 +1783,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1790,7 +1813,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-sonnet-4-6": {
|
||||
"cache_creation_input_token_cost": 4.125e-06,
|
||||
|
|
@ -1819,7 +1843,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -1880,7 +1905,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"anthropic.claude-v1": {
|
||||
"input_cost_per_token": 8e-06,
|
||||
|
|
@ -2130,7 +2156,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -2219,7 +2246,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"azure/ada": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -9360,7 +9388,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -9382,7 +9411,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-east-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
|
|
@ -9535,7 +9565,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -9557,7 +9588,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/meta.llama3-70b-instruct-v1:0": {
|
||||
"input_cost_per_token": 2.65e-06,
|
||||
|
|
@ -10287,7 +10319,8 @@
|
|||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"claude-opus-4-1": {
|
||||
"cache_creation_input_token_cost": 1.875e-05,
|
||||
|
|
@ -14095,7 +14128,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -14308,7 +14342,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0": {
|
||||
"input_cost_per_token": 1.3e-07,
|
||||
|
|
@ -19834,7 +19869,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -19886,7 +19922,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.amazon.nova-2-lite-v1:0": {
|
||||
"cache_read_input_token_cost": 7.5e-08,
|
||||
|
|
@ -23995,7 +24032,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"jp.anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.375e-06,
|
||||
|
|
@ -24018,7 +24056,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"crusoe/deepseek-ai/DeepSeek-R1-0528": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -32603,7 +32642,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0": {
|
||||
"input_cost_per_token": 3e-06,
|
||||
|
|
@ -32762,7 +32802,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": {
|
||||
"cache_creation_input_token_cost": 4.5e-06,
|
||||
|
|
@ -32789,7 +32830,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"au.anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.375e-06,
|
||||
|
|
@ -32811,7 +32853,8 @@
|
|||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true
|
||||
"supports_native_structured_output": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.875e-05,
|
||||
|
|
@ -32865,7 +32908,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"global.anthropic.claude-opus-4-5-20251101-v1:0": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -32894,7 +32938,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"eu.anthropic.claude-opus-4-5-20251101-v1:0": {
|
||||
"cache_creation_input_token_cost": 6.25e-06,
|
||||
|
|
@ -32922,7 +32967,8 @@
|
|||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "high"
|
||||
"bedrock_output_config_effort_ceiling": "high",
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0": {
|
||||
"cache_creation_input_token_cost": 3.75e-06,
|
||||
|
|
@ -42996,7 +43042,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true
|
||||
"supports_pdf_input": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": {
|
||||
"cache_creation_input_token_cost": 1.5e-06,
|
||||
|
|
@ -43019,7 +43066,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true
|
||||
"supports_pdf_input": true,
|
||||
"supports_parallel_tool_use_config": true
|
||||
},
|
||||
"snowflake/claude-sonnet-4-5": {
|
||||
"max_tokens": 16384,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -2608,102 +2609,132 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5():
|
|||
TTL ordering constraint (tools -> system -> messages).
|
||||
|
||||
Ref: https://github.com/BerriAI/litellm/issues/XXXXX
|
||||
|
||||
Forces the bundled local cost map so ttl eligibility (driven by
|
||||
`cache_creation_input_token_cost_above_1hr` in litellm.model_cost) reads
|
||||
this branch's pricing data rather than the network-fetched `main` copy,
|
||||
which lacks the fix until merge.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
add_cache_point_tool_block,
|
||||
)
|
||||
|
||||
tool_with_1h = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
tool_with_1h = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
|
||||
# Claude 4.5 model: ttl should be preserved
|
||||
result = add_cache_point_tool_block(
|
||||
tool_with_1h, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result is not None
|
||||
assert result["cachePoint"]["type"] == "default"
|
||||
assert result["cachePoint"]["ttl"] == "1h"
|
||||
# Claude 4.5 model: ttl should be preserved
|
||||
result = add_cache_point_tool_block(
|
||||
tool_with_1h, model="jp.anthropic.claude-opus-4-7"
|
||||
)
|
||||
assert result is not None
|
||||
assert result["cachePoint"]["type"] == "default"
|
||||
assert result["cachePoint"]["ttl"] == "1h"
|
||||
|
||||
# Claude 4.5 model with 5m ttl: also preserved
|
||||
tool_with_5m = {
|
||||
"cache_control": {"type": "ephemeral", "ttl": "5m"},
|
||||
}
|
||||
result_5m = add_cache_point_tool_block(
|
||||
tool_with_5m, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result_5m is not None
|
||||
assert result_5m["cachePoint"]["ttl"] == "5m"
|
||||
# Claude 4.5 model with 5m ttl: also preserved
|
||||
tool_with_5m = {
|
||||
"cache_control": {"type": "ephemeral", "ttl": "5m"},
|
||||
}
|
||||
result_5m = add_cache_point_tool_block(
|
||||
tool_with_5m, model="jp.anthropic.claude-opus-4-7"
|
||||
)
|
||||
assert result_5m is not None
|
||||
assert result_5m["cachePoint"]["ttl"] == "5m"
|
||||
|
||||
# Older model: ttl should be stripped
|
||||
result_old = add_cache_point_tool_block(
|
||||
tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
assert result_old is not None
|
||||
assert result_old["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_old["cachePoint"]
|
||||
# Older model: ttl should be stripped
|
||||
result_old = add_cache_point_tool_block(
|
||||
tool_with_1h, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
assert result_old is not None
|
||||
assert result_old["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_old["cachePoint"]
|
||||
|
||||
# No model provided: ttl should be stripped (safe default)
|
||||
result_no_model = add_cache_point_tool_block(tool_with_1h, model=None)
|
||||
assert result_no_model is not None
|
||||
assert "ttl" not in result_no_model["cachePoint"]
|
||||
# No model provided: ttl should be stripped (safe default)
|
||||
result_no_model = add_cache_point_tool_block(tool_with_1h, model=None)
|
||||
assert result_no_model is not None
|
||||
assert "ttl" not in result_no_model["cachePoint"]
|
||||
|
||||
# No cache_control: returns None (unchanged behavior)
|
||||
tool_no_cache = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
}
|
||||
assert add_cache_point_tool_block(tool_no_cache) is None
|
||||
# No cache_control: returns None (unchanged behavior)
|
||||
tool_no_cache = {
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
}
|
||||
assert add_cache_point_tool_block(tool_no_cache) is None
|
||||
|
||||
# cache_control without ttl: returns default cachePoint (unchanged behavior)
|
||||
tool_no_ttl = {"cache_control": {"type": "ephemeral"}}
|
||||
result_no_ttl = add_cache_point_tool_block(
|
||||
tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
assert result_no_ttl is not None
|
||||
assert result_no_ttl["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_no_ttl["cachePoint"]
|
||||
# cache_control without ttl: returns default cachePoint (unchanged behavior)
|
||||
tool_no_ttl = {"cache_control": {"type": "ephemeral"}}
|
||||
result_no_ttl = add_cache_point_tool_block(
|
||||
tool_no_ttl, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
)
|
||||
assert result_no_ttl is not None
|
||||
assert result_no_ttl["cachePoint"]["type"] == "default"
|
||||
assert "ttl" not in result_no_ttl["cachePoint"]
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_bedrock_tools_pt_passes_ttl_for_claude_4_5():
|
||||
"""
|
||||
End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl
|
||||
for Claude 4.5+ models when tools have cache_control with ttl.
|
||||
|
||||
Forces the bundled local cost map so ttl eligibility (driven by
|
||||
`cache_creation_input_token_cost_above_1hr` in litellm.model_cost) reads
|
||||
this branch's pricing data rather than the network-fetched `main` copy,
|
||||
which lacks the fix until merge.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import _bedrock_tools_pt
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
]
|
||||
"cache_control": {"type": "ephemeral", "ttl": "1h"},
|
||||
}
|
||||
]
|
||||
|
||||
# Claude 4.5: cachePoint should have ttl
|
||||
result = _bedrock_tools_pt(
|
||||
tools, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
)
|
||||
cache_blocks = [b for b in result if "cachePoint" in b]
|
||||
assert len(cache_blocks) == 1
|
||||
assert cache_blocks[0]["cachePoint"]["ttl"] == "1h"
|
||||
# Claude 4.5: cachePoint should have ttl
|
||||
result = _bedrock_tools_pt(tools, model="jp.anthropic.claude-opus-4-7")
|
||||
cache_blocks = [b for b in result if "cachePoint" in b]
|
||||
assert len(cache_blocks) == 1
|
||||
assert cache_blocks[0]["cachePoint"]["ttl"] == "1h"
|
||||
|
||||
# Older model: cachePoint should not have ttl
|
||||
result_old = _bedrock_tools_pt(
|
||||
tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
cache_blocks_old = [b for b in result_old if "cachePoint" in b]
|
||||
assert len(cache_blocks_old) == 1
|
||||
assert "ttl" not in cache_blocks_old[0]["cachePoint"]
|
||||
# Older model: cachePoint should not have ttl
|
||||
result_old = _bedrock_tools_pt(
|
||||
tools, model="anthropic.claude-3-5-sonnet-20241022-v2:0"
|
||||
)
|
||||
cache_blocks_old = [b for b in result_old if "cachePoint" in b]
|
||||
assert len(cache_blocks_old) == 1
|
||||
assert "ttl" not in cache_blocks_old[0]["cachePoint"]
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document():
|
||||
|
|
|
|||
|
|
@ -611,6 +611,65 @@ def test_transform_request_helper_includes_anthropic_beta_and_tools():
|
|||
assert fields["tools"][0]["type"] == "computer_20250124"
|
||||
|
||||
|
||||
def test_parallel_tool_calls_config_kept_for_flagged_model():
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
config = AmazonConverseConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
model="anthropic.claude-sonnet-4-6",
|
||||
non_default_params={"parallel_tool_calls": False},
|
||||
optional_params={},
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
data = config._transform_request_helper(
|
||||
model="anthropic.claude-sonnet-4-6",
|
||||
system_content_blocks=[],
|
||||
optional_params=optional_params,
|
||||
messages=None,
|
||||
)
|
||||
|
||||
assert data["additionalModelRequestFields"]["tool_choice"] == {
|
||||
"disable_parallel_tool_use": True
|
||||
}
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_parallel_tool_calls_config_dropped_for_ttl_only_model(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
model = "anthropic.claude-fable-5"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"cache_creation_input_token_cost_above_1hr": 2e-05},
|
||||
)
|
||||
config = AmazonConverseConfig()
|
||||
optional_params = config.map_openai_params(
|
||||
model=model,
|
||||
non_default_params={"parallel_tool_calls": False},
|
||||
optional_params={},
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
data = config._transform_request_helper(
|
||||
model=model,
|
||||
system_content_blocks=[],
|
||||
optional_params=optional_params,
|
||||
messages=None,
|
||||
)
|
||||
|
||||
assert "tool_choice" not in data.get("additionalModelRequestFields", {})
|
||||
|
||||
|
||||
def test_transform_response_with_computer_use_tool():
|
||||
"""Test response transformation with computer use tool call."""
|
||||
import httpx
|
||||
|
|
@ -4098,8 +4157,10 @@ _TOOL_PARAM = [
|
|||
]
|
||||
|
||||
|
||||
def test_parallel_tool_calls_newer_model_adds_disable_flag():
|
||||
def test_parallel_tool_calls_newer_model_adds_disable_flag(monkeypatch):
|
||||
"""Newer Claude models (4.5+) should get disable_parallel_tool_use in additionalModelRequestFields."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
|
||||
config = AmazonConverseConfig()
|
||||
model = "anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}]
|
||||
|
|
@ -4130,6 +4191,44 @@ def test_parallel_tool_calls_newer_model_adds_disable_flag():
|
|||
assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"]
|
||||
|
||||
|
||||
def test_parallel_tool_calls_flag_decoupled_from_ttl_pricing(monkeypatch):
|
||||
"""
|
||||
The disable_parallel_tool_use gate must read supports_parallel_tool_use_config,
|
||||
not the 1h-TTL pricing field: a model carrying only the former still gets the flag.
|
||||
"""
|
||||
from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock
|
||||
|
||||
config = AmazonConverseConfig()
|
||||
model = "anthropic.claude-parallel-tool-use-only"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost, model, {"supports_parallel_tool_use_config": True}
|
||||
)
|
||||
assert is_claude_4_5_on_bedrock(model) is False
|
||||
messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}]
|
||||
|
||||
optional_params = config.map_openai_params(
|
||||
non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
request_data = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert (
|
||||
request_data["additionalModelRequestFields"]["tool_choice"][
|
||||
"disable_parallel_tool_use"
|
||||
]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def test_parallel_tool_calls_older_model_drops_disable_flag():
|
||||
"""Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it."""
|
||||
config = AmazonConverseConfig()
|
||||
|
|
@ -4554,6 +4653,166 @@ def test_cache_control_injection_tool_config_not_added_without_injection_point()
|
|||
assert all("cachePoint" not in tool for tool in tools)
|
||||
|
||||
|
||||
def test_cache_control_injection_tool_config_honors_ttl_for_supported_model():
|
||||
"""
|
||||
Regression test: cache_control_injection_points with location=tool_config
|
||||
must honor the requested `control.ttl`, mirroring the message/system
|
||||
cache_control behavior, instead of always emitting a bare
|
||||
{"type": "default"} cachePoint with no ttl.
|
||||
|
||||
Forces the bundled local cost map so `is_claude_4_5_on_bedrock` (which
|
||||
reads `cache_creation_input_token_cost_above_1hr` from litellm.model_cost)
|
||||
sees this branch's pricing data rather than the network-fetched `main`
|
||||
copy, which lacks it until merge.
|
||||
"""
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
config = AmazonConverseConfig()
|
||||
messages = [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
]
|
||||
optional_params = {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"cache_control_injection_points": [
|
||||
{
|
||||
"location": "tool_config",
|
||||
"control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
],
|
||||
}
|
||||
result = config._transform_request(
|
||||
model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
)
|
||||
tools = result["toolConfig"]["tools"]
|
||||
assert tools[-1] == {"cachePoint": {"type": "default", "ttl": "1h"}}
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_cache_control_injection_tool_config_honors_ttl_for_regional_model_lacking_own_pricing():
|
||||
"""
|
||||
Regression test: a regional pricing entry that omits
|
||||
`cache_creation_input_token_cost_above_1hr` (e.g. `jp.anthropic.claude-opus-4-7`)
|
||||
must not shadow the base model entry that carries it; the requested ttl
|
||||
survives through the base-model fallback.
|
||||
"""
|
||||
old_env = os.environ.get("LITELLM_LOCAL_MODEL_COST_MAP")
|
||||
old_cost = litellm.model_cost
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
try:
|
||||
assert (
|
||||
"cache_creation_input_token_cost_above_1hr"
|
||||
not in litellm.model_cost["jp.anthropic.claude-opus-4-7"]
|
||||
)
|
||||
assert (
|
||||
"cache_creation_input_token_cost_above_1hr"
|
||||
in litellm.model_cost["anthropic.claude-opus-4-7"]
|
||||
)
|
||||
config = AmazonConverseConfig()
|
||||
messages = [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
]
|
||||
optional_params = {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"cache_control_injection_points": [
|
||||
{
|
||||
"location": "tool_config",
|
||||
"control": {"type": "ephemeral", "ttl": "1h"},
|
||||
},
|
||||
],
|
||||
}
|
||||
result = config._transform_request(
|
||||
model="jp.anthropic.claude-opus-4-7",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
)
|
||||
tools = result["toolConfig"]["tools"]
|
||||
assert tools[-1] == {"cachePoint": {"type": "default", "ttl": "1h"}}
|
||||
finally:
|
||||
litellm.model_cost = old_cost
|
||||
if old_env is None:
|
||||
os.environ.pop("LITELLM_LOCAL_MODEL_COST_MAP", None)
|
||||
else:
|
||||
os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = old_env
|
||||
|
||||
|
||||
def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model():
|
||||
"""
|
||||
Models that don't support extended TTL caching (only Claude 4.5+ on
|
||||
Bedrock does) must fall back to the default cachePoint with no ttl,
|
||||
even if the caller requested one, matching message/system behavior.
|
||||
"""
|
||||
config = AmazonConverseConfig()
|
||||
messages = [
|
||||
{"role": "user", "content": "What is the weather?"},
|
||||
]
|
||||
optional_params = {
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get weather for a location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
"cache_control_injection_points": [
|
||||
{"location": "tool_config", "control": {"type": "ephemeral", "ttl": "1h"}},
|
||||
],
|
||||
}
|
||||
result = config._transform_request(
|
||||
model="anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
messages=messages,
|
||||
optional_params=optional_params,
|
||||
litellm_params={},
|
||||
)
|
||||
tools = result["toolConfig"]["tools"]
|
||||
assert tools[-1] == {"cachePoint": {"type": "default"}}
|
||||
|
||||
|
||||
def test_translate_response_format_json_schema_still_injects_tool():
|
||||
"""
|
||||
response_format with an explicit json_schema should still use the
|
||||
|
|
|
|||
|
|
@ -25,6 +25,23 @@ from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_tran
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def local_model_cost_map(monkeypatch):
|
||||
"""Force the bundled backup cost map so capability detection reads this
|
||||
branch's flags, which the network-fetched ``main`` copy may lack."""
|
||||
import litellm
|
||||
|
||||
original = litellm.model_cost
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
litellm.model_cost = litellm.get_model_cost_map(url="")
|
||||
litellm.get_model_info.cache_clear()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
litellm.model_cost = original
|
||||
litellm.get_model_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bedrock_sse_wrapper_encodes_dict_chunks():
|
||||
"""Verify that `bedrock_sse_wrapper` converts dictionary chunks to properly formatted Server-Sent Events and forwards non-dict chunks unchanged."""
|
||||
|
|
@ -467,7 +484,7 @@ def test_bedrock_invoke_messages_transform_converts_custom_tool_schema_type_to_o
|
|||
assert result["tools"][0]["type"] == "custom"
|
||||
|
||||
|
||||
def test_remove_ttl_from_cache_control_processes_tools():
|
||||
def test_remove_ttl_from_cache_control_processes_tools(local_model_cost_map):
|
||||
"""
|
||||
Ensure _remove_ttl_from_cache_control also sanitizes cache_control on tools.
|
||||
|
||||
|
|
@ -513,7 +530,9 @@ def test_remove_ttl_from_cache_control_processes_tools():
|
|||
assert "ttl" not in request["system"][0]["cache_control"]
|
||||
|
||||
|
||||
def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5():
|
||||
def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5(
|
||||
local_model_cost_map,
|
||||
):
|
||||
"""
|
||||
For Claude 4.5+ models, ttl in ["5m", "1h"] should be preserved on tools,
|
||||
just like it is for system and messages.
|
||||
|
|
@ -539,7 +558,7 @@ def test_remove_ttl_from_cache_control_preserves_tools_ttl_for_claude_4_5():
|
|||
}
|
||||
|
||||
cfg._remove_ttl_from_cache_control(
|
||||
request, model="us.anthropic.claude-sonnet-4-5-20250514-v1:0"
|
||||
request, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0"
|
||||
)
|
||||
|
||||
# Both tools and system should preserve ttl for Claude 4.5
|
||||
|
|
|
|||
|
|
@ -263,3 +263,33 @@ def test_bundled_bedrock_opus_model_info_declares_output_config_effort_ceiling(
|
|||
model_info = GetModelCostMap.load_local_model_cost_map()[model]
|
||||
|
||||
assert model_info["bedrock_output_config_effort_ceiling"] == expected_ceiling
|
||||
|
||||
|
||||
def test_capability_lookups_fall_back_to_base_model_when_regional_entry_lacks_field(
|
||||
monkeypatch,
|
||||
):
|
||||
"""
|
||||
Regression test: a regional model_cost entry without the capability field
|
||||
must not shadow a base entry that has it (`get(model) or get(base)` used to
|
||||
short-circuit on the truthy regional dict and drop the capability).
|
||||
"""
|
||||
import litellm
|
||||
from litellm.llms.bedrock.common_utils import (
|
||||
bedrock_converse_supports_parallel_tool_use_config,
|
||||
is_claude_4_5_on_bedrock,
|
||||
)
|
||||
|
||||
base = "anthropic.claude-fallback-test"
|
||||
regional = f"eu.{base}"
|
||||
monkeypatch.setitem(litellm.model_cost, regional, {"input_cost_per_token": 1e-06})
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
base,
|
||||
{
|
||||
"cache_creation_input_token_cost_above_1hr": 1e-05,
|
||||
"supports_parallel_tool_use_config": True,
|
||||
},
|
||||
)
|
||||
|
||||
assert is_claude_4_5_on_bedrock(regional) is True
|
||||
assert bedrock_converse_supports_parallel_tool_use_config(regional) is True
|
||||
|
|
|
|||
|
|
@ -843,6 +843,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
|
|||
"supports_image_input": {"type": "boolean"},
|
||||
"supports_nova_canvas_image_edit": {"type": "boolean"},
|
||||
"supports_parallel_function_calling": {"type": "boolean"},
|
||||
"supports_parallel_tool_use_config": {"type": "boolean"},
|
||||
"supports_pdf_input": {"type": "boolean"},
|
||||
"supports_prompt_caching": {"type": "boolean"},
|
||||
"supports_response_schema": {"type": "boolean"},
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue