diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index e6402e8c1bd..ba59e3fa997 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -4957,10 +4957,13 @@ def make_valid_bedrock_tool_name(input_tool_name: str) -> str: def add_cache_point_tool_block(tool: dict, model: str | None = None) -> BedrockToolBlock | None: - from litellm.llms.bedrock.common_utils import is_claude_4_5_on_bedrock + from litellm.llms.bedrock.common_utils import ( + bedrock_model_accepts_cache_points, + is_claude_4_5_on_bedrock, + ) cache_control: Final = tool.get("cache_control", None) - if cache_control is not None: + if cache_control is not None and bedrock_model_accepts_cache_points(model): cache_point: Final = cache_control.get("type", "ephemeral") if cache_point == "ephemeral": cache_point_block: Final[CachePointBlock] = {"type": "default"} diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 7fefeaeaf04..5363c3c0366 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -87,6 +87,7 @@ from ..common_utils import ( BedrockError, BedrockModelInfo, bedrock_converse_supports_parallel_tool_use_config, + bedrock_model_accepts_cache_points, get_anthropic_beta_from_headers, get_bedrock_tool_name, is_bedrock_application_inference_profile_arn, @@ -1149,7 +1150,7 @@ class AmazonConverseConfig(BaseConfig): model: str | None = None, ) -> SystemContentBlock | ContentBlock | None: cache_control: Final = message_block.get("cache_control", None) - if cache_control is None: + if cache_control is None or not bedrock_model_accepts_cache_points(model): return None cache_point: Final = self._build_cache_point_block(cache_control, model) @@ -1613,7 +1614,7 @@ class AmazonConverseConfig(BaseConfig): # Append cachePoint to tools if cache_control_injection_points has tool_config cache_injection_points: Final = additional_request_params.pop("cache_control_injection_points", None) - if cache_injection_points and len(bedrock_tools) > 0: + if cache_injection_points and len(bedrock_tools) > 0 and bedrock_model_accepts_cache_points(model): for point in cache_injection_points: if point.get("location") == "tool_config": cache_point = self._build_cache_point_block(point.get("control"), model) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 30a77d57f24..66ee5f10679 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -816,6 +816,30 @@ def bedrock_converse_supports_parallel_tool_use_config(model: str) -> bool: ) +def bedrock_model_accepts_cache_points(model: str | None) -> bool: + """ + Whether Converse ``cachePoint`` blocks may be sent to this model. + + Bedrock rejects requests carrying cachePoint blocks for models without prompt + caching support ("You invoked an unsupported model or your request did not allow + prompt caching"), so a model whose cost-map entry does not declare + ``supports_prompt_caching`` must not receive them. A model absent from the map + (an application inference profile ARN, a model newer than the map) keeps emitting + so existing caching setups never silently degrade. ``litellm.utils.supports_prompt_caching`` + is not reusable here: it returns False for unmapped models, the opposite polarity. + """ + if model is None: + return True + entries: Final = tuple( + entry + for candidate in (model, get_bedrock_base_model(model)) + if (entry := litellm.model_cost.get(candidate)) is not None + ) + if not entries: + return True + return any(entry.get("supports_prompt_caching") is True for entry in entries) + + def is_claude_4_5_on_bedrock(model: str) -> bool: """ Check if the model supports Bedrock prompt caching with an extended '1h' TTL diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 64c96b575c5..dd2d45f00c6 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -2932,6 +2932,28 @@ def test_add_cache_point_tool_block_passes_ttl_for_claude_4_5(monkeypatch): monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", old_env) +def test_add_cache_point_tool_block_stands_down_for_model_without_prompt_caching(monkeypatch): + """A tool carrying cache_control must not become a cachePoint for a Bedrock model + whose cost-map entry lacks prompt caching support, since Bedrock rejects the whole + request. An unmapped id keeps emitting so ARN deployments do not lose caching.""" + from litellm.litellm_core_utils.prompt_templates.factory import ( + add_cache_point_tool_block, + ) + + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + tool = {"cache_control": {"type": "ephemeral"}} + + assert add_cache_point_tool_block(tool, model="nvidia.nemotron-super-3-120b") is None + assert add_cache_point_tool_block(tool, model="us.nvidia.nemotron-super-3-120b") is None + assert add_cache_point_tool_block( + tool, model="arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123" + ) == {"cachePoint": {"type": "default"}} + assert add_cache_point_tool_block(tool, model="us.anthropic.claude-sonnet-4-5-20250929-v1:0") == { + "cachePoint": {"type": "default"} + } + + def test_bedrock_tools_pt_passes_ttl_for_claude_4_5(monkeypatch): """ End-to-end: _bedrock_tools_pt should produce cachePoint blocks with ttl diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 4f53d3481de..70f3153ed7e 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -5248,6 +5248,84 @@ def test_cache_control_injection_tool_config_drops_ttl_for_unsupported_model(): assert tools[-1] == {"cachePoint": {"type": "default"}} +@pytest.mark.parametrize( + ("model", "expects_cache_points"), + [ + pytest.param("nvidia.nemotron-super-3-120b", False, id="mapped-model-without-prompt-caching"), + pytest.param("us.nvidia.nemotron-super-3-120b", False, id="regional-prefix-resolves-through-base-model"), + pytest.param( + "us.anthropic.claude-3-5-sonnet-20240620-v1:0", False, id="claude-named-but-not-caching-on-bedrock" + ), + pytest.param("us.anthropic.claude-sonnet-4-5-20250929-v1:0", True, id="mapped-model-with-prompt-caching"), + pytest.param( + "arn:aws:bedrock:us-east-1:123456789012:application-inference-profile/abc123", + True, + id="unmapped-arn-keeps-emitting", + ), + ], +) +def test_cache_points_emitted_only_for_models_that_support_prompt_caching(model, expects_cache_points, monkeypatch): + """Bedrock rejects cachePoint blocks for models without prompt caching support + ("You invoked an unsupported model or your request did not allow prompt caching"), + and clients like Claude Code attach cache_control to every request, so a map-known + model without the capability must not receive them. Unmapped ids (application + inference profile ARNs, models newer than the map) keep emitting so existing + caching setups never silently degrade.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + body = AmazonConverseConfig().transform_request( + model=model, + messages=[ + {"role": "system", "content": [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert ("cachePoint" in json.dumps(body)) is expects_cache_points + assert body["system"][0]["text"] == "sys" + assert body["messages"][0]["content"][0]["text"] == "hi" + + +def test_tool_config_cachepoint_not_placed_or_credited_for_model_without_prompt_caching(monkeypatch): + """The tool_config injection point must stand down with the rest of the cachePoint + emission when the model cannot cache, and spend attribution must not credit the + gateway for a breakpoint that was never placed.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + + bucket: dict = {"user_api_key": "sk-test"} + data = AmazonConverseConfig()._transform_request_helper( + model="nvidia.nemotron-super-3-120b", + system_content_blocks=[], + optional_params={ + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + }, + } + ], + "cache_control_injection_points": [{"location": "tool_config"}], + }, + messages=[{"role": "user", "content": "hi"}], + litellm_params={"metadata": bucket, "litellm_metadata": None, "model_info": {"id": "dep-bedrock"}}, + ) + + assert "cachePoint" not in json.dumps(data.get("toolConfig", {})) + assert "litellm_gateway_injected_cache" not in bucket + + def test_translate_response_format_json_schema_still_injects_tool(): """ response_format with an explicit json_schema should still use the @@ -6211,7 +6289,7 @@ def test_message_level_cache_control_drops_ttl_for_unsupported_model(ttl_target) result = _bedrock_converse_messages_pt( messages=_agentic_messages_with_ttl(ttl_target), - model="anthropic.claude-3-5-sonnet-20240620-v1:0", + model="anthropic.claude-3-5-sonnet-20241022-v2:0", llm_provider="bedrock_converse", )