diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index e4638909dbd..580372bdac4 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -367,10 +367,7 @@ _ALLOW_CLIENT_MESSAGE_REDACTION_OPT_OUT_METADATA_KEY: Final = "allow_client_mess _CLIENT_PRICING_CONTROL_FIELDS: Final = frozenset(CustomPricingLiteLLMParams.model_fields.keys()) | {"base_model"} # ``model_info`` carries the same pricing fields when read by # ``use_custom_pricing_for_model``; strip from metadata for the same reason. -# ``standard_logging_guardrail_information`` is proxy-written telemetry summed -# into response_cost and spend; a client seeding it forges (even negative) -# guardrail cost. -_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logging_guardrail_information"}) +_CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info"}) # ``attempted_fallbacks`` and ``original_model_group`` are written by the router # and read by spend logs as fact; a client value has no legitimate meaning and no # key or team setting keeps it, so the strip is never gated. @@ -626,6 +623,19 @@ def _strip_client_pricing_overrides(data: dict[str, object]) -> None: ) +def strip_client_guardrail_information(data: dict[str, object]) -> None: + """Guardrail hooks write this field and its cost is billed, so no key or team setting keeps a caller's value.""" + for metadata_key in ("metadata", "litellm_metadata"): + if ( + isinstance(metadata := data.get(metadata_key), dict) + and "standard_logging_guardrail_information" in metadata + ): + metadata.pop("standard_logging_guardrail_information") + verbose_proxy_logger.debug( + "Stripped client-supplied guardrail information from request body: %s", metadata_key + ) + + def _strip_router_reserved_metadata( data: dict[str, Any], # mutable-ok: strips in place on the request body the pre-call pipeline threads through ) -> None: @@ -2239,6 +2249,7 @@ async def add_litellm_data_to_request( # would silently skip the field. if not _key_or_team_allows_client_pricing_override(user_api_key_dict): _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) _strip_router_reserved_metadata(data) # Same reason as the strips above: runs after the metadata string-to-dict parse diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 18e70f3b446..c31933c2a91 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -102,6 +102,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above + strip_client_guardrail_information, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import normalize_route_for_root_path @@ -1133,6 +1134,9 @@ async def pass_through_request( _parsed_body, ) + if _parsed_body is not None: + strip_client_guardrail_information(_parsed_body) + ### COLLECT GUARDRAILS FOR PASSTHROUGH ENDPOINT ### # Passthrough endpoints are opt-in only for guardrails # When enabled, collect guardrails from org/team/key levels + passthrough-specific diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 40b79449315..039627694a8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3738,6 +3738,7 @@ class ProxyLogging: key: { **_as_str_mapping(existing.get(key)), **_as_str_mapping(value), + **user_api_key_logged_metadata, } for key, value in _litellm_params.items() if key == "litellm_metadata" diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 12faf3bde47..a6cb35fdcac 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -1343,6 +1343,39 @@ async def test_pass_through_request_uses_resolved_timeout(): assert mock_get_client.call_args[1]["params"]["timeout"] == 1500 +@pytest.mark.asyncio +@pytest.mark.parametrize("allow_client_pricing_override", [True, False]) +async def test_pass_through_request_strips_client_guardrail_information(allow_client_pricing_override: bool): + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({}) + mock_request.query_params = QueryParams({}) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=RuntimeError("stop before upstream")) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + with pytest.raises(ProxyException): + await pass_through_request( + request=mock_request, + target="http://test.com", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth( + metadata={"allow_client_pricing_override": allow_client_pricing_override} + ), + custom_body={ + "model": "gpt-4o", + "input_cost_per_token": 0.5, + "metadata": { + "trace_id": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + }, + ) + + hooked_body: Final = mock_proxy_logging.pre_call_hook.call_args.kwargs["data"] + assert hooked_body["metadata"] == {"trace_id": "keep-me"} + + @pytest.mark.asyncio async def test_create_pass_through_route_forwards_timeout(): unique_path = "/test/path/unique/timeout" diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.py index a77620994dc..670fcc58755 100644 --- a/tests/unit/proxy/test_pricing_field_strip.py +++ b/tests/unit/proxy/test_pricing_field_strip.py @@ -22,6 +22,7 @@ from litellm.proxy.litellm_pre_call_utils import ( _CLIENT_PRICING_METADATA_FIELDS, _strip_client_pricing_overrides, add_litellm_data_to_request, + strip_client_guardrail_information, ) from litellm.types.utils import CustomPricingLiteLLMParams @@ -136,7 +137,7 @@ class TestStripClientPricingOverrides: "standard_logging_guardrail_information": [{"guardrail_cost": 5.0}], }, } - _strip_client_pricing_overrides(data) + strip_client_guardrail_information(data) assert data["metadata"] == {"user_session": "keep-me"} assert data["litellm_metadata"] == {} @@ -167,7 +168,6 @@ class TestStripClientPricingOverrides: def test_metadata_field_set_contains_model_info(self): assert "model_info" in _CLIENT_PRICING_METADATA_FIELDS - assert "standard_logging_guardrail_information" in _CLIENT_PRICING_METADATA_FIELDS def test_strip_emits_debug_log_listing_dropped_fields(self, caplog): # Operators need a paper trail so they can diagnose why a previously @@ -300,6 +300,33 @@ async def test_add_litellm_data_to_request_skips_strip_with_key_opt_in(): assert updated["base_model"] == "gpt-4o-mini" +@pytest.mark.asyncio +@pytest.mark.parametrize("opt_in_field", ["metadata", "team_metadata"]) +async def test_add_litellm_data_to_request_strips_guardrail_information_despite_pricing_opt_in(opt_in_field): + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "input_cost_per_token": 0.0001, + "metadata": { + "user_session": "keep-me", + "standard_logging_guardrail_information": [{"guardrail_name": "forged", "guardrail_cost": 5.0}], + }, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_make_request_mock(), + user_api_key_dict=_user_api_key_auth(**{opt_in_field: {"allow_client_pricing_override": True}}), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["input_cost_per_token"] == 0.0001 + assert updated["metadata"]["user_session"] == "keep-me" + assert "standard_logging_guardrail_information" not in updated["metadata"] + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_json_string_litellm_metadata(): """``litellm_metadata`` may arrive as a JSON-encoded string (multipart/ diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 71fb3de7f17..4dbabfccf89 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -186,6 +186,7 @@ async def test_proxy_only_error_log_keeps_tags_and_drops_caller_logging_fields( assert "caller" not in str(details["litellm_params"]) assert details["litellm_params"][metadata_field]["tags"] == ["t"] assert details["litellm_params"]["metadata"]["user_api_key_team_id"] == "team-a" + assert details["litellm_params"][metadata_field]["user_api_key_team_id"] == "team-a" @pytest.mark.asyncio