diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index d2fbb26bb02..300834073f7 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -29,6 +29,7 @@ if TYPE_CHECKING: from websockets.asyncio.client import ClientConnection from websockets.exceptions import ConnectionClosed + from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks CLIENT_CONNECTION_CLASS = ClientConnection @@ -123,6 +124,12 @@ DefaultLoggedRealTimeEventTypes: Final = [ ] +def _as_user_api_key_auth(user_api_key_dict: object) -> "UserAPIKeyAuth | None": + from litellm.proxy._types import UserAPIKeyAuth + + return user_api_key_dict if isinstance(user_api_key_dict, UserAPIKeyAuth) else None + + class RealTimeStreaming: def __init__( self, @@ -831,6 +838,7 @@ class RealTimeStreaming: typed user messages and tool outputs use ``pre_call``. """ from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.types.guardrails import GuardrailEventHooks if event_hooks is None: @@ -852,7 +860,12 @@ class RealTimeStreaming: try: await callback.apply_guardrail( inputs={"texts": [transcript], "images": []}, - request_data={"user_api_key_dict": self.user_api_key_dict}, + request_data={ + "user_api_key_dict": self.user_api_key_dict, + "litellm_metadata": BaseTranslation.transform_user_api_key_dict_to_metadata( + _as_user_api_key_auth(self.user_api_key_dict) + ), + }, input_type="request", ) except Exception as e: diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 89ad67f0485..1a772ca8ee3 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -81,43 +81,17 @@ class BaseTranslation(ABC): @staticmethod def transform_user_api_key_dict_to_metadata( - user_api_key_dict: Any | None, + user_api_key_dict: "UserAPIKeyAuth | None", ) -> dict[str, object]: - """ - Transform user_api_key_dict to a metadata dict with prefixed keys. - - Converts keys like 'user_id' to 'user_api_key_user_id' to clearly indicate - the source of the metadata. - - Args: - user_api_key_dict: UserAPIKeyAuth object or dict with user information - - Returns: - Dict with keys prefixed with 'user_api_key_' - """ + """The authenticated key's identity as prefixed metadata, an allowlist safe to hand to guardrail vendors.""" if user_api_key_dict is None: return {} + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup - # Convert to dict if it's a Pydantic object - user_dict = user_api_key_dict.model_dump() if hasattr(user_api_key_dict, "model_dump") else user_api_key_dict - - if not isinstance(user_dict, dict): - return {} - - # Transform keys to be prefixed with 'user_api_key_' - transformed: Final[dict[str, object]] = {} - for key, value in user_dict.items(): - # Skip None values and internal fields - if value is None or key.startswith("_"): - continue - - # If key already has the prefix, use as-is, otherwise add prefix - if key.startswith("user_api_key_"): - transformed[key] = value - else: - transformed[f"user_api_key_{key}"] = value - - return transformed + return { + **LiteLLMProxyRequestSetup.get_authenticated_identity_metadata(user_api_key_dict), + "user_api_key_key_alias": user_api_key_dict.key_alias, + } @staticmethod def merge_user_api_key_metadata_into_request( diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 1f295a6e656..ac7dcfd4f39 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -20,6 +20,8 @@ if TYPE_CHECKING: from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.utils import ProxyLogging +_PROXY_OWNED_PAYLOAD_KEYS: Final = frozenset({"metadata", "litellm_metadata", "litellm_logging_obj"}) + class PassThroughEndpointHandler(BaseTranslation): """ @@ -80,7 +82,7 @@ class PassThroughEndpointHandler(BaseTranslation): from litellm.litellm_core_utils.safe_json_dumps import safe_dumps payload_to_check: Final = { - k: v for k, v in data.items() if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj") + k: v for k, v in data.items() if not k.startswith("_") and k not in _PROXY_OWNED_PAYLOAD_KEYS } verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail") return safe_dumps(payload_to_check) diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 6053ab26726..18d62142503 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -34,6 +34,7 @@ from litellm.proxy.guardrails.guardrail_hooks.custom_code.sandbox import ( ) from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router +from litellm.proxy.litellm_pre_call_utils import caller_metadata_with_authenticated_identity from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import GuardrailsRepository @@ -2404,9 +2405,14 @@ async def apply_guardrail( if litellm_logging_obj is not None: _patch_logging_obj_for_guardrail(litellm_logging_obj, request) + processed_metadata: Final = data.get("metadata") + inbound_headers: Final = processed_metadata.get("headers") if isinstance(processed_metadata, dict) else None request_data: Final[dict] = { **({"messages": request.messages} if request.messages is not None else {}), - **({"metadata": request.metadata} if request.metadata is not None else {}), + "metadata": { + **caller_metadata_with_authenticated_identity(request.metadata, user_api_key_dict), + **({"headers": inbound_headers} if inbound_headers is not None else {}), + }, } _input_type: Final = _resolve_guardrail_input_type(active_guardrail, request.input_type) guardrailed_inputs: Final = await active_guardrail.apply_guardrail( diff --git a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py index 3d1a173635e..320682c89b7 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py +++ b/litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py @@ -292,9 +292,8 @@ class GenericGuardrailAPI(CustomGuardrail): if value is not None: result_metadata[field_name] = value - # handle user_api_key_token = user_api_key_hash - if metadata_dict.get("user_api_key_token") is not None: - result_metadata["user_api_key_hash"] = metadata_dict.get("user_api_key_token") + if litellm_metadata.get("user_api_key_token") is not None and "user_api_key_hash" not in result_metadata: + result_metadata["user_api_key_hash"] = litellm_metadata["user_api_key_token"] verbose_proxy_logger.debug( "Generic Guardrail API: Extracted user metadata: %s", diff --git a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py index df5a265bb72..143e5818ee8 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py +++ b/litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py @@ -1740,16 +1740,6 @@ class PanwPrismaAirsHandler(CustomGuardrail): call_id, _mcp_tool, ) - elif not request_data and logging_obj is None and input_type == "request": - # Direct /apply_guardrail endpoint — empty request_data, no - # logging_obj. Existing behavior: synthesize UUID. - call_id = str(uuid.uuid4()) - request_data["litellm_call_id"] = call_id - verbose_proxy_logger.warning( - "PANW Prisma AIRS: litellm_call_id missing from empty " - "request_data, synthesized %s (direct /apply_guardrail?)", - call_id, - ) else: call_id = str(uuid.uuid4()) request_data["litellm_call_id"] = call_id diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index d68a55f9a88..eeb604e98af 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -155,16 +155,25 @@ def _a2a_jsonrpc_error_chunk(exc: HTTPException, request_id: str | None) -> Mapp } -def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> None: - """Populate data['litellm_metadata'] from user_api_key_dict if absent.""" - if "litellm_metadata" not in data: - from litellm.llms.base_llm.guardrail_translation.base_translation import ( - BaseTranslation, - ) +_PROXY_ENRICHED_IDENTITY_FIELDS: Final = frozenset({"user_api_key_auth_metadata"}) - user_metadata: Final = BaseTranslation.transform_user_api_key_dict_to_metadata(user_api_key_dict) - if user_metadata: - data["litellm_metadata"] = user_metadata + +def _ensure_litellm_metadata(data: dict, user_api_key_dict: UserAPIKeyAuth) -> None: + """Overwrite the identity fields of data['litellm_metadata'] from the authenticated key, in place.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import ( + BaseTranslation, + ) + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + existing: Final = data.get("litellm_metadata") + if isinstance(existing, dict): + identity: Final = LiteLLMProxyRequestSetup.get_authenticated_identity_metadata(user_api_key_dict) + existing.update({key: value for key, value in identity.items() if key not in _PROXY_ENRICHED_IDENTITY_FIELDS}) + existing.pop("user_api_key_token", None) + return + user_metadata: Final = BaseTranslation.transform_user_api_key_dict_to_metadata(user_api_key_dict) + if user_metadata: + data["litellm_metadata"] = user_metadata class UnifiedLLMGuardrails(CustomLogger): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 866d84ca8f2..9201fcd7ca7 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -493,6 +493,40 @@ def _strip_untrusted_request_header_controls( headers.pop(header_name, None) +def is_untrusted_caller_metadata_key(key: str) -> bool: + return key.startswith("user_api_key_") or key in _UNTRUSTED_METADATA_CONTROL_FIELDS + + +def strip_untrusted_caller_metadata( + data: MutableMapping[str, object], *, allow_client_message_redaction_opt_out: bool +) -> None: + """Remove, in place, the proxy-owned slots a caller put in either metadata bucket of a request body.""" + for user_meta in (data.get("metadata"), data.get("litellm_metadata")): + if not isinstance(user_meta, dict): + continue + _strip_untrusted_request_header_controls( + user_meta.get("headers"), + allow_client_message_redaction_opt_out=allow_client_message_redaction_opt_out, + ) + for untrusted_key in tuple(key for key in user_meta if is_untrusted_caller_metadata_key(key)): + user_meta.pop(untrusted_key, None) + + +_GUARDRAIL_UNTRUSTED_CALLER_METADATA_KEYS: Final = frozenset({"user_api_key", "headers"}) + + +def caller_metadata_with_authenticated_identity( + caller_metadata: Mapping[str, object] | None, user_api_key_dict: UserAPIKeyAuth +) -> dict[str, object]: + """Caller metadata minus proxy-owned slots, bare user_api_key and headers, with the key's identity on top.""" + caller_fields: Final = { + key: value + for key, value in (caller_metadata or {}).items() + if not (is_untrusted_caller_metadata_key(key) or key in _GUARDRAIL_UNTRUSTED_CALLER_METADATA_KEYS) + } + return {**caller_fields, **LiteLLMProxyRequestSetup.get_authenticated_identity_metadata(user_api_key_dict)} + + def _is_false_like(value: object) -> bool: if isinstance(value, bool): return value is False @@ -520,7 +554,7 @@ def _key_or_team_allows_client_mock_response( ) -def _key_or_team_allows_client_message_redaction_opt_out( +def key_or_team_allows_client_message_redaction_opt_out( user_api_key_dict: UserAPIKeyAuth, ) -> bool: return _key_or_team_metadata_flag_is_true( @@ -1642,6 +1676,24 @@ class LiteLLMProxyRequestSetup: ) return user_api_key_logged_metadata + @staticmethod + def get_key_scoped_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + return { + "user_api_key_metadata": strip_callback_config(user_api_key_dict.metadata), + "user_api_key_team_metadata": strip_callback_config(user_api_key_dict.team_metadata), + "user_api_key_object_permission_id": user_api_key_dict.object_permission_id, + "user_api_key_team_object_permission_id": user_api_key_dict.team_object_permission_id, + } + + @staticmethod + def get_authenticated_identity_metadata(user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + """Identity fields derived from the authenticated key alone, for paths that skip the chat-path build.""" + return { + **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict), + "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(user_api_key_dict), + **LiteLLMProxyRequestSetup.get_key_scoped_metadata(user_api_key_dict), + } + @staticmethod def add_user_api_key_auth_to_request_metadata( data: dict, @@ -2001,7 +2053,7 @@ async def add_litellm_data_to_request( # These keys are injected by the proxy itself below — user-supplied values # must not be trusted. _allow_client_mock_response: Final = _key_or_team_allows_client_mock_response(user_api_key_dict) - _allow_client_message_redaction_opt_out = _key_or_team_allows_client_message_redaction_opt_out(user_api_key_dict) + _allow_client_message_redaction_opt_out = key_or_team_allows_client_message_redaction_opt_out(user_api_key_dict) for _internal_key in _UNTRUSTED_ROOT_CONTROL_FIELDS: if _allow_client_mock_response and _internal_key in _CLIENT_MOCK_CONTROL_FIELDS: continue @@ -2183,31 +2235,12 @@ async def add_litellm_data_to_request( # profile_id) don't see attacker-injected admin slots preserved in # the deepcopy. - # Strip internal pipeline state and admin-injection slots from user input. # Runs AFTER the string-to-dict parse above so JSON-string metadata (sent # via multipart/form-data or extra_body) cannot smuggle admin fields past # the isinstance(dict) guard. - # - # The proxy populates a family of ``user_api_key_*`` fields below - # (user_api_key_metadata, user_api_key_user_id, user_api_key_alias, - # user_api_key_spend, user_api_key_team_metadata, …) into - # data[_metadata_variable_name]. Because the proxy only writes to ONE of - # the two metadata dicts, a caller pre-populating any of these keys on - # the OTHER metadata dict would have their forged values surface in - # guardrails, spend tracking, audit logs, and identity resolution. Strip - # by prefix so new ``user_api_key_*`` fields added in the future are - # covered without per-key maintenance. - for _meta_key in ("metadata", "litellm_metadata"): - _user_meta = data.get(_meta_key) - if isinstance(_user_meta, dict): - _strip_untrusted_request_header_controls( - _user_meta.get("headers"), - allow_client_message_redaction_opt_out=(_allow_client_message_redaction_opt_out), - ) - for _k in [ - k for k in _user_meta if k.startswith("user_api_key_") or k in _UNTRUSTED_METADATA_CONTROL_FIELDS - ]: - _user_meta.pop(_k, None) + strip_untrusted_caller_metadata( + data, allow_client_message_redaction_opt_out=_allow_client_message_redaction_opt_out + ) # Strip pricing overrides AFTER the litellm_metadata string-to-dict parse # above, for the same reason as the user_api_key_* strip — JSON-string @@ -2384,14 +2417,7 @@ async def add_litellm_data_to_request( data[_metadata_variable_name]["user_api_key_user_model_max_budget"] = user_model_budget # rebind-ok: out-param data[_metadata_variable_name].update(carried_budget_metadata(user_api_key_dict)) - data[_metadata_variable_name]["user_api_key_metadata"] = strip_callback_config(user_api_key_dict.metadata) - data[_metadata_variable_name]["user_api_key_team_metadata"] = strip_callback_config(user_api_key_dict.team_metadata) - data[_metadata_variable_name]["user_api_key_object_permission_id"] = getattr( - user_api_key_dict, "object_permission_id", None - ) - data[_metadata_variable_name]["user_api_key_team_object_permission_id"] = getattr( - user_api_key_dict, "team_object_permission_id", None - ) + data[_metadata_variable_name].update(LiteLLMProxyRequestSetup.get_key_scoped_metadata(user_api_key_dict)) data[_metadata_variable_name]["headers"] = _logging_safe_headers data[_metadata_variable_name]["endpoint"] = str(request.url) # Carry the proxy-receive instant via metadata (like `endpoint`) so the diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index e0a4184291e..c1cd652ceba 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -100,6 +100,8 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above + key_or_team_allows_client_message_redaction_opt_out, + strip_untrusted_caller_metadata, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.utils import normalize_route_for_root_path @@ -1120,6 +1122,18 @@ async def pass_through_request( _parsed_body = {} else: _parsed_body = await _read_request_body(request) + strip_untrusted_caller_metadata( + _parsed_body, + allow_client_message_redaction_opt_out=key_or_team_allows_client_message_redaction_opt_out( + user_api_key_dict + ), + ) + # Guardrails forward these to vendors as the inbound request headers; all are popped before the upstream send. + _parsed_body.pop("proxy_server_request", None) + _parsed_body.pop("headers", None) + for _caller_bucket in (_parsed_body.get("metadata"), _parsed_body.get("litellm_metadata")): + if isinstance(_caller_bucket, dict): + _caller_bucket.pop("headers", None) verbose_proxy_logger.debug( "Pass through endpoint sending request to \nURL %s\nheaders: %s\nbody: %s\n", url, diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py index a5e79f84ef1..3f77a24b707 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_generic_guardrail_api.py @@ -416,6 +416,22 @@ class TestMetadataExtraction: assert request_metadata["user_api_key_hash"] == "hashed-token-value" assert request_metadata["user_api_key_user_id"] == "test-user" + @pytest.mark.parametrize( + "request_data, expected_hash", + [ + pytest.param({"metadata": {"user_api_key_token": "caller-token"}}, None, id="caller-bucket-token-ignored"), + pytest.param( + {"litellm_metadata": {"user_api_key_token": "proxy-token", "user_api_key_hash": "logged-key"}}, + "logged-key", + id="hash-wins-over-token", + ), + ], + ) + def test_token_fallback_only_from_litellm_metadata_and_only_without_hash( + self, generic_guardrail, request_data, expected_hash + ): + assert generic_guardrail._extract_user_api_key_metadata(request_data).get("user_api_key_hash") == expected_hash + @pytest.mark.asyncio async def test_metadata_extraction_empty_when_no_metadata(self, generic_guardrail): """Test metadata extraction returns empty dict when no metadata available""" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py index 53af7f36a5f..cce13844201 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_grayswan.py @@ -582,15 +582,20 @@ def test_ensure_litellm_metadata_populates_from_user_api_key_dict() -> None: assert data["litellm_metadata"]["user_api_key_team_id"] == "t1" -def test_ensure_litellm_metadata_noop_when_already_present() -> None: - """Verify _ensure_litellm_metadata does not overwrite existing litellm_metadata.""" +def test_ensure_litellm_metadata_overrides_caller_identity_in_existing_bucket() -> None: + """An existing litellm_metadata keeps its other keys, but its identity comes from the authenticated key.""" from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( _ensure_litellm_metadata, ) - user_auth = UserAPIKeyAuth(user_id="should-not-appear") - data: dict = {"litellm_metadata": {"existing": "value"}} + user_auth = UserAPIKeyAuth(user_id="auth-user", key_alias="auth-alias", team_id="auth-team") + bucket: dict = {"existing": "value", "user_api_key_alias": "batch-worker", "user_api_key_team_id": "team-exempt"} + data: dict = {"litellm_metadata": bucket} _ensure_litellm_metadata(data, user_auth) - assert data["litellm_metadata"] == {"existing": "value"} + assert data["litellm_metadata"] is bucket + assert bucket["existing"] == "value" + assert bucket["user_api_key_alias"] == "auth-alias" + assert bucket["user_api_key_team_id"] == "auth-team" + assert bucket["user_api_key_user_id"] == "auth-user" diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index c90f88ec110..9b9fa0b89fb 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -1,6 +1,7 @@ """Tests for unified guardrail.""" import logging +from collections.abc import Callable from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal @@ -2395,3 +2396,228 @@ class TestTranslationMappingsAreReadLive: assert not [ name for name, value in vars(unified_module).items() if isinstance(value, dict) and CallTypes.aocr in value ] + + +_RAW_CLI_SESSION_TOKEN: Final = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa" + + +def _sk_key(route: str) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + api_key="sk-real-caller-key", + key_alias="prod-app", + team_id="team-prod", + metadata={"key_label": "k1"}, + team_metadata={"phoenix_project_name": "team-proj", "priority": "high"}, + request_route=route, + ) + + +def _cli_session_key(route: str) -> UserAPIKeyAuth: + return UserAPIKeyAuth( + token=_RAW_CLI_SESSION_TOKEN, + key_alias="cli-session-alice", + user_id="alice", + is_session_token=True, + team_id="team-prod", + team_metadata={"phoenix_project_name": "team-proj", "priority": "high"}, + request_route=route, + ) + + +class TestGuardrailsSeeAuthenticatedIdentity: + """A request body cannot make a guardrail vendor see another key's identity, and the real one reaches it.""" + + @staticmethod + def _generic_guardrail(vendor_payloads: list[dict[str, object]]) -> CustomGuardrail: + import json + + import httpx + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI + + def vendor(request: httpx.Request) -> httpx.Response: + vendor_payloads.append(json.loads(request.content)) + return httpx.Response(200, json={"action": "NONE"}) + + guardrail = GenericGuardrailAPI(api_base="https://guardrail.test", guardrail_name="generic") + guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(vendor)) + return guardrail + + @pytest.mark.asyncio + @pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"]) + async def test_pass_through_body_cannot_forge_identity(self, monkeypatch, bucket: str) -> None: + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + vendor_payloads: list[dict[str, object]] = [] + key = UserAPIKeyAuth(api_key="sk-real-caller-key", key_alias="prod-app", team_id="team-prod") + data = { + "guardrail_to_apply": self._generic_guardrail(vendor_payloads), + "prompt": "hello", + bucket: { + "user_api_key_alias": "batch-worker", + "user_api_key_team_id": "team-exempt", + "user_api_key_token": "forged-hash", + }, + } + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + + assert len(vendor_payloads) == 1 + identity = vendor_payloads[0]["request_data"] + assert identity["user_api_key_alias"] == "prod-app" + assert identity["user_api_key_team_id"] == "team-prod" + assert identity["user_api_key_hash"] == key.api_key + + @pytest.mark.asyncio + async def test_mcp_tool_call_reaches_vendor_with_key_alias(self) -> None: + from litellm.proxy.utils import ProxyLogging + + vendor_payloads: list[dict[str, object]] = [] + key = UserAPIKeyAuth(api_key="sk-real-caller-key", key_alias="prod-app", team_id="team-prod") + proxy_logging = ProxyLogging(user_api_key_cache=DualCache()) + mcp_kwargs = { + "name": "search", + "arguments": {"query": "hello"}, + "server_name": "docs", + "user_api_key_auth": key, + "user_api_key_user_id": key.user_id, + "user_api_key_team_id": key.team_id, + "user_api_key_end_user_id": None, + "user_api_key_hash": key.api_key, + "headers": {}, + } + data = proxy_logging._convert_mcp_to_llm_format( + proxy_logging._create_mcp_request_object_from_kwargs(mcp_kwargs), mcp_kwargs + ) + data["guardrail_to_apply"] = self._generic_guardrail(vendor_payloads) + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.call_mcp_tool.value + ) + + assert len(vendor_payloads) == 1 + identity = vendor_payloads[0]["request_data"] + assert identity["user_api_key_alias"] == "prod-app" + assert identity["user_api_key_team_id"] == "team-prod" + assert identity["user_api_key_hash"] == key.api_key + + @pytest.mark.asyncio + async def test_pass_through_body_cannot_forge_request_route(self, monkeypatch) -> None: + """Guardrails and call-type lookups key on user_api_key_request_route, so it must be the key's own route.""" + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + key = UserAPIKeyAuth(api_key="sk-real-caller-key", request_route="/openai/v1/chat/completions") + data = { + "guardrail_to_apply": RecordingGuardrail(), + "prompt": "hello", + "litellm_metadata": {"user_api_key_request_route": "/v1/embeddings"}, + } + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + + assert data["litellm_metadata"]["user_api_key_request_route"] == "/openai/v1/chat/completions" + + @pytest.mark.asyncio + @pytest.mark.parametrize("route", ["/v1/messages", "/v1/responses", "/v1/chat/completions"]) + @pytest.mark.parametrize( + "make_key", [pytest.param(_sk_key, id="sk-key"), pytest.param(_cli_session_key, id="cli-session-key")] + ) + async def test_chat_path_request_keeps_proxy_metadata_and_sends_stable_hash( + self, monkeypatch, route: str, make_key: Callable[[str], UserAPIKeyAuth] + ) -> None: + """After the chat-path metadata build, the guardrail hook leaves the proxy's bucket as it was (team metadata + in user_api_key_auth_metadata included) and the vendor gets the logged key, never a raw CLI session token.""" + import copy + import json + from unittest.mock import MagicMock + + from fastapi import Request + + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_litellm_data_to_request + + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + request = MagicMock(spec=Request) + request.url = MagicMock() + request.url.path = route + request.url.__str__.return_value = "http://localhost" + route + request.method = "POST" + request.query_params = {} + request.headers = {"Content-Type": "application/json"} + request.client = MagicMock() + request.client.host = "127.0.0.1" + request.state = MagicMock() + key = make_key(route) + data = await add_litellm_data_to_request( + data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + request=request, + user_api_key_dict=key, + proxy_config=MagicMock(), + general_settings={}, + version="v", + ) + proxy_bucket = data.get("litellm_metadata") + unshared = ("litellm_parent_otel_span", "user_api_key_auth") + bucket_before = ( + copy.deepcopy({k: v for k, v in proxy_bucket.items() if k not in unshared}) if proxy_bucket else None + ) + vendor_payloads: list[dict[str, object]] = [] + data["guardrail_to_apply"] = self._generic_guardrail(vendor_payloads) + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.acompletion.value + ) + + assert len(vendor_payloads) == 1 + assert vendor_payloads[0]["request_data"]["user_api_key_hash"] == LiteLLMProxyRequestSetup.get_logged_api_key( + key + ) + assert _RAW_CLI_SESSION_TOKEN not in json.dumps(vendor_payloads[0]) + if bucket_before is not None: + assert data["litellm_metadata"] is proxy_bucket + assert {k: v for k, v in proxy_bucket.items() if k in bucket_before} == bucket_before + assert bucket_before["user_api_key_auth_metadata"]["priority"] == "high" + assert "user_api_key_token" not in proxy_bucket + + @pytest.mark.asyncio + @pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata", None]) + async def test_pass_through_cli_session_key_sends_stable_hash(self, monkeypatch, bucket: str | None) -> None: + import json + + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + vendor_payloads: list[dict[str, object]] = [] + key = _cli_session_key("/anthropic/v1/messages") + forged_bucket = {bucket: {"user_api_key_token": "forged-hash"}} if bucket else {} + data = {"guardrail_to_apply": self._generic_guardrail(vendor_payloads), "prompt": "hello", **forged_bucket} + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + + assert len(vendor_payloads) == 1 + assert vendor_payloads[0]["request_data"]["user_api_key_hash"] == "cli-session-alice" + assert _RAW_CLI_SESSION_TOKEN not in json.dumps(vendor_payloads[0]) + assert vendor_payloads[0]["texts"] == ['{"prompt": "hello"}'] + + @pytest.mark.asyncio + async def test_token_only_key_drops_forged_token_already_in_proxy_bucket(self, monkeypatch) -> None: + """A key with no api_key logs no hash, so a user_api_key_token left in litellm_metadata would become the + vendor's hash if the hook kept it.""" + _patch_translation_mappings(monkeypatch, discover_guardrail_translation_mappings()) + vendor_payloads: list[dict[str, object]] = [] + key = UserAPIKeyAuth(token="abc123hashed", key_alias="prod-app") + data = { + "guardrail_to_apply": self._generic_guardrail(vendor_payloads), + "prompt": "hello", + "litellm_metadata": {"user_api_key_token": "forged-hash"}, + } + + await UnifiedLLMGuardrails().async_pre_call_hook( + user_api_key_dict=key, cache=DualCache(), data=data, call_type=CallTypes.pass_through.value + ) + + assert len(vendor_payloads) == 1 + assert "user_api_key_token" not in data["litellm_metadata"] + assert vendor_payloads[0]["request_data"].get("user_api_key_hash") is None diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py index 508736fb78e..8a82fbdc377 100644 --- a/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/test_litellm/proxy/guardrails/test_guardrail_endpoints.py @@ -1487,12 +1487,12 @@ async def test_apply_guardrail_invokes_logging_pipeline(mocker): } -def _patch_apply_guardrail_env(mocker, guardrail_result): +def _patch_apply_guardrail_env(mocker, guardrail_result, processed_data=None, guardrail=None): mock_guardrail = mocker.Mock() mock_guardrail.apply_guardrail = AsyncMock(return_value=guardrail_result) mock_registry = mocker.Mock() - mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail + mock_registry.get_initialized_guardrail_callback.return_value = guardrail or mock_guardrail mocker.patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY", mock_registry) mock_logging_obj = mocker.Mock() @@ -1500,7 +1500,7 @@ def _patch_apply_guardrail_env(mocker, guardrail_result): mock_logging_obj.model_call_details = {} mock_processor = mocker.Mock() mock_processor.common_processing_pre_call_logic = AsyncMock( - return_value=({"guardrail_name": "test-guardrail"}, mock_logging_obj) + return_value=(processed_data or {"guardrail_name": "test-guardrail"}, mock_logging_obj) ) mocker.patch( "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", @@ -1535,11 +1535,12 @@ async def test_apply_guardrail_forwards_metadata_to_guardrail(mocker): user_api_key_dict=UserAPIKeyAuth(), ) - mock_guardrail.apply_guardrail.assert_awaited_once_with( - inputs={"texts": ["What are tax loopholes?"]}, - request_data={"metadata": {"forbidden_topics": ["tax"]}}, - input_type="request", - ) + mock_guardrail.apply_guardrail.assert_awaited_once() + call = mock_guardrail.apply_guardrail.await_args.kwargs + assert call["inputs"] == {"texts": ["What are tax loopholes?"]} + assert call["input_type"] == "request" + assert "messages" not in call["request_data"] + assert call["request_data"]["metadata"]["forbidden_topics"] == ["tax"] @pytest.mark.asyncio @@ -1561,39 +1562,211 @@ async def test_apply_guardrail_forwards_metadata_and_messages_together(mocker): user_api_key_dict=UserAPIKeyAuth(), ) - mock_guardrail.apply_guardrail.assert_awaited_once_with( - inputs={"texts": ["What are tax loopholes?"]}, - request_data={ - "messages": messages, - "metadata": {"forbidden_topics": ["tax"]}, - }, - input_type="request", - ) + request_data = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"] + assert request_data["messages"] == messages + assert request_data["metadata"]["forbidden_topics"] == ["tax"] @pytest.mark.asyncio -async def test_apply_guardrail_omits_metadata_when_not_sent(mocker): - """Without metadata, request_data stays empty (backward-compatible).""" +async def test_apply_guardrail_authenticated_identity_overrides_client_metadata(mocker): + """A caller must not be able to claim another key's or team's identity in the body metadata.""" + mock_guardrail = _patch_apply_guardrail_env(mocker, {"texts": ["ok"]}) + caller = UserAPIKeyAuth( + api_key="sk-real-caller-key", + key_alias="real-caller", + team_id="real-team", + user_id="real-user", + ) + + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", + text="hello", + metadata={ + "user_api_key_alias": "exempt-batch-worker", + "user_api_key_team_id": "exempt-team", + "user_api_key_user_id": "someone-else", + "user_api_key_hash": "forged-hash", + "forbidden_topics": ["tax"], + }, + ) + await apply_guardrail(fastapi_request=mocker.Mock(), request=request, user_api_key_dict=caller) + + metadata = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"]["metadata"] + assert metadata["user_api_key_alias"] == "real-caller" + assert metadata["user_api_key_team_id"] == "real-team" + assert metadata["user_api_key_user_id"] == "real-user" + assert metadata["user_api_key_hash"] == caller.api_key + assert metadata["user_api_key_hash"] != "forged-hash" + assert metadata["forbidden_topics"] == ["tax"] + + +@pytest.mark.asyncio +async def test_apply_guardrail_drops_client_identity_fields_the_key_does_not_set(mocker): + """Proxy-owned slots in the body never reach the guardrail, including user_api_key_token, which guardrails + map onto the key hash, and control fields the chat path also strips.""" + mock_guardrail = _patch_apply_guardrail_env(mocker, {"texts": ["ok"]}) + caller = UserAPIKeyAuth(metadata={"zguard_policy_id": "strict"}, object_permission_id="perm-real") + + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", + text="hello", + metadata={ + "user_api_key_alias": "exempt-batch-worker", + "user_api_key_team_id": "exempt-team", + "user_api_key_token": "forged-hash", + "user_api_key_metadata": {"zguard_policy_id": "permissive"}, + "user_api_key_object_permission_id": "perm-forged", + "user_api_key": "forged-key", + "applied_guardrails": ["already-ran"], + "headers": {"x-end-user": "someone-else"}, + "trace_label": "nightly", + }, + ) + await apply_guardrail(fastapi_request=mocker.Mock(), request=request, user_api_key_dict=caller) + + metadata = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"]["metadata"] + assert metadata["user_api_key_alias"] is None + assert metadata["user_api_key_team_id"] is None + assert "user_api_key_token" not in metadata + assert metadata["user_api_key_metadata"] == {"zguard_policy_id": "strict"} + assert metadata["user_api_key_object_permission_id"] == "perm-real" + assert metadata["user_api_key"] is None + assert "applied_guardrails" not in metadata + assert "headers" not in metadata + assert metadata["trace_label"] == "nightly" + + +@pytest.mark.asyncio +async def test_apply_guardrail_forwards_real_request_headers_not_caller_supplied_ones(mocker): + """Guardrails forward metadata headers to vendors, so they must be the proxy's view of the request.""" + real_headers = {"user-agent": "real-client/1.0"} + mock_guardrail = _patch_apply_guardrail_env( + mocker, + {"texts": ["ok"]}, + processed_data={"guardrail_name": "test-guardrail", "metadata": {"headers": real_headers}}, + ) + + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", + text="hello", + metadata={"headers": {"user-agent": "forged/1.0", "x-end-user": "someone-else"}}, + ) + await apply_guardrail(fastapi_request=mocker.Mock(), request=request, user_api_key_dict=UserAPIKeyAuth()) + + metadata = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"]["metadata"] + assert metadata["headers"] == real_headers + + +@pytest.mark.asyncio +async def test_apply_guardrail_generic_guardrail_api_sends_authenticated_identity_to_vendor(mocker): + """End to end through a real GenericGuardrailAPI: the vendor payload names the authenticated key even when + the body forges user_api_key_alias and user_api_key_token, which the generic guardrail maps onto the hash.""" + import httpx + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI + + vendor_payloads = [] + + def vendor(request: httpx.Request) -> httpx.Response: + vendor_payloads.append(json.loads(request.content)) + return httpx.Response(200, json={"action": "NONE"}) + + generic_guardrail = GenericGuardrailAPI(api_base="https://guardrail.test", guardrail_name="generic") + generic_guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(vendor)) + _patch_apply_guardrail_env(mocker, {"texts": ["unused"]}, guardrail=generic_guardrail) + caller = UserAPIKeyAuth(api_key="sk-real-caller-key", key_alias="real-caller", team_id="real-team") + + request = ApplyGuardrailRequest( + guardrail_name="generic", + text="hello", + metadata={"user_api_key_alias": "exempt-batch-worker", "user_api_key_token": "forged-hash"}, + ) + response = await apply_guardrail(fastapi_request=mocker.Mock(), request=request, user_api_key_dict=caller) + + assert response.response_text == "hello" + assert len(vendor_payloads) == 1 + identity = vendor_payloads[0]["request_data"] + assert identity["user_api_key_alias"] == "real-caller" + assert identity["user_api_key_team_id"] == "real-team" + assert identity["user_api_key_hash"] == caller.api_key + + +@pytest.mark.asyncio +async def test_apply_guardrail_request_route_comes_from_the_key(mocker): + """Guardrails pick call-type behavior from user_api_key_request_route, so the body cannot choose it.""" + mock_guardrail = _patch_apply_guardrail_env(mocker, {"texts": ["ok"]}) + + request = ApplyGuardrailRequest( + guardrail_name="test-guardrail", + text="hello", + metadata={"user_api_key_request_route": "/v1/embeddings"}, + ) + await apply_guardrail( + fastapi_request=mocker.Mock(), + request=request, + user_api_key_dict=UserAPIKeyAuth(request_route="/guardrails/apply_guardrail"), + ) + + metadata = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"]["metadata"] + assert metadata["user_api_key_request_route"] == "/guardrails/apply_guardrail" + + +@pytest.mark.asyncio +async def test_apply_guardrail_cli_session_key_sends_stable_hash_to_vendor(mocker): + """A CLI session key's raw per-login token must never reach the vendor; it gets the stable logged key.""" + import httpx + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI + + raw_session_token = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa" + vendor_payloads = [] + + def vendor(request: httpx.Request) -> httpx.Response: + vendor_payloads.append(json.loads(request.content)) + return httpx.Response(200, json={"action": "NONE"}) + + generic_guardrail = GenericGuardrailAPI(api_base="https://guardrail.test", guardrail_name="generic") + generic_guardrail.async_handler = AsyncHTTPHandler(transport=httpx.MockTransport(vendor)) + _patch_apply_guardrail_env(mocker, {"texts": ["unused"]}, guardrail=generic_guardrail) + caller = UserAPIKeyAuth( + token=raw_session_token, key_alias="cli-session-alice", user_id="alice", is_session_token=True + ) + + request = ApplyGuardrailRequest( + guardrail_name="generic", text="hello", metadata={"user_api_key_token": raw_session_token} + ) + await apply_guardrail(fastapi_request=mocker.Mock(), request=request, user_api_key_dict=caller) + + assert len(vendor_payloads) == 1 + assert vendor_payloads[0]["request_data"]["user_api_key_hash"] == "cli-session-alice" + assert raw_session_token not in json.dumps(vendor_payloads[0]) + + +@pytest.mark.asyncio +async def test_apply_guardrail_carries_authenticated_identity_when_no_metadata_sent(mocker): + """request_data always carries the authenticated identity, even when the body has no metadata.""" mock_guardrail = _patch_apply_guardrail_env(mocker, {"texts": ["ok"]}) request = ApplyGuardrailRequest(guardrail_name="test-guardrail", text="hello") await apply_guardrail( fastapi_request=mocker.Mock(), request=request, - user_api_key_dict=UserAPIKeyAuth(), + user_api_key_dict=UserAPIKeyAuth(key_alias="known-caller", team_id="known-team"), ) - mock_guardrail.apply_guardrail.assert_awaited_once_with( - inputs={"texts": ["hello"]}, - request_data={}, - input_type="request", - ) + call = mock_guardrail.apply_guardrail.await_args.kwargs + assert call["inputs"] == {"texts": ["hello"]} + assert "messages" not in call["request_data"] + assert call["request_data"]["metadata"]["user_api_key_alias"] == "known-caller" + assert call["request_data"]["metadata"]["user_api_key_team_id"] == "known-team" @pytest.mark.asyncio async def test_apply_guardrail_forwards_explicit_empty_messages_and_metadata(mocker): - """Explicitly-sent empty messages/metadata must be forwarded, not dropped; - only omitted fields stay out of request_data.""" + """Explicitly-sent empty messages must be forwarded, not dropped, and empty + metadata still carries the authenticated identity.""" mock_guardrail = _patch_apply_guardrail_env(mocker, {"texts": ["ok"]}) request = ApplyGuardrailRequest( @@ -1605,14 +1778,12 @@ async def test_apply_guardrail_forwards_explicit_empty_messages_and_metadata(moc await apply_guardrail( fastapi_request=mocker.Mock(), request=request, - user_api_key_dict=UserAPIKeyAuth(), + user_api_key_dict=UserAPIKeyAuth(key_alias="known-caller"), ) - mock_guardrail.apply_guardrail.assert_awaited_once_with( - inputs={"texts": ["hello"]}, - request_data={"messages": [], "metadata": {}}, - input_type="request", - ) + request_data = mock_guardrail.apply_guardrail.await_args.kwargs["request_data"] + assert request_data["messages"] == [] + assert request_data["metadata"]["user_api_key_alias"] == "known-caller" @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py index 3469df082e0..652acfbe5b9 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -7622,3 +7622,81 @@ def test_passthrough_attributes_a_cli_session_to_its_alias_not_the_login_token() metadata = kwargs["litellm_params"]["metadata"] assert metadata["user_api_key"] == "cli-session-alice" assert _get_spend_logs_metadata(metadata)["user_api_key"] == "cli-session-alice" + + +@pytest.mark.asyncio +async def test_pass_through_request_strips_caller_identity_before_guardrail_hooks(): + """ + Regression: a pass-through body skips add_litellm_data_to_request, so forged user_api_key_* fields, guardrail + control fields and inbound headers reached pre_call_hook guardrails as the caller's identity. The upstream body + is unchanged because these keys never reach it. + """ + from litellm.llms.custom_httpx.http_handler import get_async_httpx_client + from litellm.types.llms.custom_http import httpxSpecialProvider + + upstream_bodies = [] + + def transport_handler(upstream_request: httpx.Request) -> httpx.Response: + upstream_bodies.append(json.loads(upstream_request.content)) + return httpx.Response(200, json={"ok": True}) + + real_handler = get_async_httpx_client( + llm_provider=httpxSpecialProvider.PassThroughEndpoint, + params={"timeout": resolve_pass_through_request_timeout(None)}, + ) + cache_dict = litellm.in_memory_llm_clients_cache.cache_dict + cache_key = next(key for key, cached in cache_dict.items() if cached is real_handler) + cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler))) + + hook_data = [] + + def record_hook_data(user_api_key_dict, data, call_type): + hook_data.append({key: value for key, value in data.items() if key != "litellm_logging_obj"}) + return data + + mock_proxy_logging = MagicMock() + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=record_hook_data) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + + forged = { + "user_api_key_alias": "batch-worker", + "user_api_key_team_id": "team-exempt", + "user_api_key_token": "forged-hash", + "user_api_key_request_route": "/v1/embeddings", + "disable_global_guardrails": True, + "headers": {"x-authenticated-user": "admin@corp"}, + "trace_label": "nightly", + } + forged_headers = {"x-authenticated-user": "admin@corp", "x-litellm-end-user-id": "victim"} + body = { + "prompt": "hello", + "metadata": forged, + "litellm_metadata": forged, + "headers": forged_headers, + "proxy_server_request": {"headers": forged_headers}, + } + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + mock_request.body = AsyncMock(return_value=json.dumps(body).encode()) + + try: + with patch( + "litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging + ): # test-quality-ok: read at call time + response = await pass_through_request( + request=mock_request, + target="https://upstream.test/v1/generate", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-real-caller-key", key_alias="prod-app"), + ) + finally: + cache_dict[cache_key] = real_handler + + assert response.status_code == 200 + assert hook_data == [ + {"prompt": "hello", "metadata": {"trace_label": "nightly"}, "litellm_metadata": {"trace_label": "nightly"}} + ] + assert upstream_bodies == [{"prompt": "hello"}] diff --git a/tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py b/tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py index 4f44a4adeed..bab9958dc5d 100644 --- a/tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py +++ b/tests/unit/enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py @@ -61,11 +61,11 @@ async def test_apply_guardrail_endpoint_returns_correct_response( assert response.response_text == "Redacted text: [REDACTED] and [REDACTED]" # Verify the guardrail was called with correct parameters - mock_guardrail.apply_guardrail.assert_called_once_with( - inputs={"texts": ["Test text with PII"]}, - request_data={}, - input_type="request", - ) + mock_guardrail.apply_guardrail.assert_called_once() + call = mock_guardrail.apply_guardrail.call_args.kwargs + assert call["inputs"] == {"texts": ["Test text with PII"]} + assert call["input_type"] == "request" + assert call["request_data"]["metadata"]["user_api_key_hash"] == user_api_key_dict.api_key @pytest.mark.asyncio @@ -197,6 +197,8 @@ async def test_apply_guardrail_endpoint_without_optional_params(mock_proxy_loggi assert response.response_text == "Processed text" # Verify the guardrail was called with correct parameters - mock_guardrail.apply_guardrail.assert_called_once_with( - inputs={"texts": ["Test text"]}, request_data={}, input_type="request" - ) + mock_guardrail.apply_guardrail.assert_called_once() + call = mock_guardrail.apply_guardrail.call_args.kwargs + assert call["inputs"] == {"texts": ["Test text"]} + assert call["input_type"] == "request" + assert call["request_data"]["metadata"]["user_api_key_hash"] == user_api_key_dict.api_key diff --git a/tests/unit/litellm_core_utils/test_realtime_streaming.py b/tests/unit/litellm_core_utils/test_realtime_streaming.py index 7e6d4d24905..ed9c666ac80 100644 --- a/tests/unit/litellm_core_utils/test_realtime_streaming.py +++ b/tests/unit/litellm_core_utils/test_realtime_streaming.py @@ -3550,3 +3550,134 @@ async def test_provider_bytes_are_sent_raw_after_pacing(): assert [call.args[0] for call in backend_ws.send.await_args_list] == [b"\x00\x01", '{"type":"endStream"}'] provider_config.pace_backend_send.assert_awaited_once_with(b"\x00\x01") + + +@pytest.mark.asyncio +async def test_realtime_transcript_guardrail_receives_authenticated_identity(monkeypatch: pytest.MonkeyPatch): + """Transcript guardrails get the session key's identity in litellm_metadata, as the chat path provides it.""" + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.guardrails import GuardrailEventHooks + + received_request_data = [] + + class IdentityRecordingGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + received_request_data.append(request_data) + return inputs + + monkeypatch.setattr( + litellm, + "callbacks", + [ + IdentityRecordingGuardrail( + guardrail_name="identity_recorder", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + ) + ], + ) + key = UserAPIKeyAuth(api_key="sk-real-caller-key", key_alias="prod-app", team_id="team-prod") + streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock(), user_api_key_dict=key) + + blocked = await streaming.run_realtime_guardrails("hello there") + + assert blocked is False + assert len(received_request_data) == 1 + identity = received_request_data[0]["litellm_metadata"] + assert identity["user_api_key_alias"] == "prod-app" + assert identity["user_api_key_team_id"] == "team-prod" + assert identity["user_api_key_hash"] == key.api_key + + +@pytest.mark.asyncio +async def test_realtime_grayswan_payload_carries_only_identity(monkeypatch: pytest.MonkeyPatch): + """Gray Swan forwards litellm_metadata verbatim, so realtime must hand it identity and no key secrets.""" + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import GraySwanGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + vendor_payloads: list[dict[str, object]] = [] + + class RecordingGraySwan(GraySwanGuardrail): + async def _call_grayswan_api(self, payload): + vendor_payloads.append(payload) + return {"violation": 0.0, "violated_rules": []} + + monkeypatch.setattr( + litellm, + "callbacks", + [ + RecordingGraySwan( + guardrail_name="grayswan", + api_key="test-key", + event_hook=GuardrailEventHooks.pre_call, + default_on=True, + ) + ], + ) + key = UserAPIKeyAuth( + api_key="sk-real-caller-key", + key_alias="prod-app", + team_id="team-prod", + organization_metadata={"logging": [{"callback_vars": {"langfuse_secret_key": "SECRET-ORG"}}]}, + jwt_claims={"email": "alice@corp.example", "name": "Alice Smith"}, + team_member={"user_id": "alice", "user_email": "alice@corp.example", "role": "admin"}, + ) + streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock(), user_api_key_dict=key) + + await streaming.run_realtime_guardrails("hello there", event_hooks=[GuardrailEventHooks.pre_call]) + + assert len(vendor_payloads) == 1 + vendor_metadata = vendor_payloads[0]["litellm_metadata"] + assert vendor_metadata["user_api_key_alias"] == "prod-app" + assert vendor_metadata["user_api_key_team_id"] == "team-prod" + serialized = json.dumps(vendor_metadata) + for leaked in ("SECRET-ORG", "Alice Smith", "alice@corp.example"): + assert leaked not in serialized + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "sdk_value", + [ + pytest.param({"key_alias": "forged", "team_id": "team-exempt"}, id="dict"), + pytest.param({"spend": "not-a-number"}, id="malformed-dict"), + pytest.param("sk-raw-string", id="string"), + ], +) +async def test_realtime_guardrail_gets_no_identity_from_non_auth_sdk_value( + monkeypatch: pytest.MonkeyPatch, sdk_value: object +): + """Only a proxy-authenticated UserAPIKeyAuth yields identity; an SDK-supplied value never raises or fakes one.""" + import litellm + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.types.guardrails import GuardrailEventHooks + + received_request_data: list[dict[str, object]] = [] + + class IdentityRecordingGuardrail(CustomGuardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + received_request_data.append(request_data) + return inputs + + monkeypatch.setattr( + litellm, + "callbacks", + [ + IdentityRecordingGuardrail( + guardrail_name="identity_recorder", + event_hook=GuardrailEventHooks.realtime_input_transcription, + default_on=True, + ) + ], + ) + streaming = RealTimeStreaming(MagicMock(), MagicMock(), MagicMock(), user_api_key_dict=sdk_value) + + blocked = await streaming.run_realtime_guardrails("hello there") + + assert blocked is False + assert len(received_request_data) == 1 + assert not [key for key in received_request_data[0]["litellm_metadata"] if key.startswith("user_api_key")] diff --git a/tests/unit/llms/base_llm/guardrail_translation/__init__.py b/tests/unit/llms/base_llm/guardrail_translation/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/base_llm/guardrail_translation/test_base_translation.py b/tests/unit/llms/base_llm/guardrail_translation/test_base_translation.py new file mode 100644 index 00000000000..06745c5a15f --- /dev/null +++ b/tests/unit/llms/base_llm/guardrail_translation/test_base_translation.py @@ -0,0 +1,53 @@ +import json +from typing import Final + +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.proxy._types import UserAPIKeyAuth + +RAW_SESSION_TOKEN: Final = "cli-session-Qm7xJ2kP9sLw4vT1nR8yAa" + + +def _fully_populated_session_key() -> UserAPIKeyAuth: + return UserAPIKeyAuth( + token=RAW_SESSION_TOKEN, + is_session_token=True, + key_alias="cli-session-alice", + user_id="alice", + team_id="team-prod", + org_id="org-1", + metadata={"logging": [{"callback_name": "langfuse", "callback_vars": {"langfuse_secret_key": "SECRET-KEY"}}]}, + team_metadata={"logging": [{"callback_vars": {"langfuse_secret_key": "SECRET-TEAM"}}]}, + organization_metadata={"logging": [{"callback_vars": {"langfuse_secret_key": "SECRET-ORG"}}]}, + project_metadata={"logging": [{"callback_vars": {"langfuse_secret_key": "SECRET-PROJECT"}}]}, + jwt_claims={"sub": "alice", "email": "alice@corp.example", "name": "Alice Smith"}, + team_member={"user_id": "alice", "user_email": "alice@corp.example", "role": "admin"}, + config={"internal": "proxy-config"}, + ) + + +def test_transform_emits_only_identity_never_credentials_or_callback_secrets(): + metadata = BaseTranslation.transform_user_api_key_dict_to_metadata(_fully_populated_session_key()) + serialized = json.dumps(metadata, default=str) + + assert metadata["user_api_key_alias"] == "cli-session-alice" + assert metadata["user_api_key_key_alias"] == "cli-session-alice" + assert metadata["user_api_key_hash"] == "cli-session-alice" + assert metadata["user_api_key_team_id"] == "team-prod" + assert RAW_SESSION_TOKEN not in serialized + assert "callback_vars" not in serialized + assert "SECRET" not in serialized + assert "Alice Smith" not in serialized + assert "proxy-config" not in serialized + for dropped in ( + "user_api_key_token", + "user_api_key_jwt_claims", + "user_api_key_team_member", + "user_api_key_organization_metadata", + "user_api_key_project_metadata", + "user_api_key_config", + ): + assert dropped not in metadata + + +def test_transform_of_no_key_is_empty(): + assert BaseTranslation.transform_user_api_key_dict_to_metadata(None) == {}