diff --git a/litellm/policy_templates_backup.json b/litellm/policy_templates_backup.json index 7e8e04dffbd..ef4cdcfbd33 100644 --- a/litellm/policy_templates_backup.json +++ b/litellm/policy_templates_backup.json @@ -1078,26 +1078,15 @@ { "id": "claude-code-cost-optimization", "title": "Claude Code Cost Optimization", - "description": "Reduces Claude Code API spend by blocking expensive inference modes (fast/turbo, inference_geo, and extended thinking) and automatically injecting prompt caching headers into system messages to maximize cache hit rates.", + "description": "Reduces Claude Code API spend by blocking expensive inference modes (fast/turbo, inference_geo, and extended thinking) and Anthropic-hosted tools.", "icon": "CurrencyDollarIcon", "iconColor": "text-green-500", "iconBg": "bg-green-50", "guardrails": [ - "claude-code-inject-prompt-cache", "claude-code-block-expensive-flags" ], "complexity": "Low", "guardrailDefinitions": [ - { - "guardrail_name": "claude-code-inject-prompt-cache", - "litellm_params": { - "guardrail": "claude_code_prompt_cache", - "mode": "pre_call" - }, - "guardrail_info": { - "description": "Automatically adds cache_control: {type: ephemeral} to system messages so Anthropic can cache the system prompt prefix. Only applies to Anthropic API models. Reduces cost on repeated calls that share the same system prompt." - } - }, { "guardrail_name": "claude-code-block-expensive-flags", "litellm_params": { @@ -1111,9 +1100,8 @@ ], "templateData": { "policy_name": "claude-code-cost-optimization", - "description": "Reduces Claude Code API spend by blocking fast mode, inference_geo, extended thinking, and Anthropic-hosted tools, while auto-injecting prompt caching into system messages.", + "description": "Reduces Claude Code API spend by blocking fast mode, inference_geo, extended thinking, and Anthropic-hosted tools.", "guardrails_add": [ - "claude-code-inject-prompt-cache", "claude-code-block-expensive-flags" ], "guardrails_remove": [] diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/block_hosted_tools/test_block_hosted_tools.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/block_hosted_tools/test_block_hosted_tools.py index e82f0f9e2e6..95065649185 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/block_hosted_tools/test_block_hosted_tools.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/block_hosted_tools/test_block_hosted_tools.py @@ -6,8 +6,9 @@ import pytest from fastapi import HTTPException from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.proxy.guardrails.guardrail_hooks.block_hosted_tools.guardrail import \ - BlockHostedToolsGuardrail +from litellm.proxy.guardrails.guardrail_hooks.block_hosted_tools.guardrail import ( + BlockHostedToolsGuardrail, +) class TestBlockHostedToolsGuardrail: @@ -140,6 +141,133 @@ class TestBlockHostedToolsGuardrail: ) assert result == inputs + @pytest.mark.asyncio + async def test_blocks_anthropic_text_editor_tool(self): + """Test that Anthropic text_editor_* versioned tool is blocked.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + tools = [{"type": "text_editor_20250429", "name": "str_replace_based_edit_tool"}] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + assert "disabled" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_blocks_openai_file_search(self): + """Test that OpenAI file_search hosted tool is blocked.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + tools = [{"type": "file_search"}] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + assert "disabled" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_blocks_gemini_code_execution_top_level_key(self): + """Test that Gemini native code_execution top-level key is blocked.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + tools = [{"code_execution": {}}] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + assert "disabled" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_reports_all_blocked_tools(self): + """Test that all blocked tools are reported when multiple are present.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + tools = [ + {"type": "bash_20250124", "name": "run_bash"}, + {"type": "code_interpreter"}, + ] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + detail = exc_info.value.detail + assert exc_info.value.status_code == 403 + assert len(detail["blocked_tools"]) == 2 + + @pytest.mark.asyncio + async def test_allows_tools_none(self): + """Test that tools=None passes through without error.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + inputs = {"tools": None} + request_data = {} + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + assert result == inputs + + @pytest.mark.asyncio + async def test_mixed_allowed_and_blocked_tools(self): + """Test that a mix of function tools and one blocked tool raises 403.""" + guardrail = BlockHostedToolsGuardrail( + guardrail_name="test-block-hosted-tools" + ) + tools = [ + { + "type": "function", + "function": {"name": "get_weather", "description": "Get weather"}, + }, + {"type": "web_search_20250305"}, + ] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + detail = exc_info.value.detail + assert len(detail["blocked_tools"]) == 1 + @pytest.mark.asyncio async def test_http_403_classified_as_guardrail_intervention(self): """Test that HTTP 403 from guardrail is classified as guardrail_intervened.""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/claude_code/test_block_expensive_flags.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/claude_code/test_block_expensive_flags.py index 3bb060c398b..a29f1b6aca8 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/claude_code/test_block_expensive_flags.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/claude_code/test_block_expensive_flags.py @@ -111,6 +111,99 @@ class TestClaudeCodeBlockExpensiveFlagsGuardrail: ) assert result == inputs + @pytest.mark.asyncio + async def test_allows_thinking_type_not_enabled(self): + """Test that thinking.type != 'enabled' is not blocked.""" + guardrail = ClaudeCodeBlockExpensiveFlagsGuardrail( + guardrail_name="test-block-expensive" + ) + request_data = {"thinking": {"type": "disabled"}} + inputs = {"tools": None} + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + assert result == inputs + + @pytest.mark.asyncio + async def test_allows_speed_not_fast(self): + """Test that speed != 'fast' is not blocked.""" + guardrail = ClaudeCodeBlockExpensiveFlagsGuardrail( + guardrail_name="test-block-expensive" + ) + request_data = {"speed": "normal"} + inputs = {"tools": None} + + result = await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + assert result == inputs + + @pytest.mark.asyncio + async def test_blocks_inference_geo_any_value(self): + """Test that inference_geo is blocked for any value (wildcard).""" + guardrail = ClaudeCodeBlockExpensiveFlagsGuardrail( + guardrail_name="test-block-expensive" + ) + for geo_value in ["eu", "ap", "us-east"]: + request_data = {"inference_geo": geo_value} + inputs = {"tools": None} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + + @pytest.mark.asyncio + async def test_blocks_inherited_anthropic_hosted_tool(self): + """Test that inherited Anthropic hosted tools are blocked.""" + guardrail = ClaudeCodeBlockExpensiveFlagsGuardrail( + guardrail_name="test-block-expensive" + ) + # bash_* prefix is inherited from block_hosted_tools/anthropic.yaml + tools = [{"type": "bash_20250124", "name": "run_bash"}] + inputs = {"tools": tools} + request_data = {} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + assert "bash" in str(exc_info.value.detail).lower() + + @pytest.mark.asyncio + async def test_param_block_takes_priority_over_tools(self): + """Test that a blocked param raises before tool checks.""" + guardrail = ClaudeCodeBlockExpensiveFlagsGuardrail( + guardrail_name="test-block-expensive" + ) + tools = [{"type": "bash_20250124", "name": "run_bash"}] + request_data = {"speed": "fast"} + inputs = {"tools": tools} + + with pytest.raises(HTTPException) as exc_info: + await guardrail.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="request", + ) + + assert exc_info.value.status_code == 403 + # speed=fast check fires first + assert "fast" in str(exc_info.value.detail).lower() + @pytest.mark.asyncio async def test_http_403_classified_as_guardrail_intervention(self): """Test that HTTP 403 from guardrail is classified as guardrail_intervened."""