diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 24a36a7f2dc..2682e2e329c 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -45,6 +45,7 @@ from litellm.constants import ( MAX_LITELLM_CALL_ID_LENGTH, MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, NON_INFERENCE_CALL_TYPES, + REDACTED_BY_LITELLM, RETURN_RAW_MODEL_NAME_METADATA_KEY, STREAM_SSE_DATA_PREFIX, STREAM_SSE_KEEPALIVE_PING_BYTES, @@ -128,6 +129,8 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( + can_route_with_loaded_models, + raise_if_model_not_routable, route_request, ) from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id @@ -275,6 +278,7 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.utils import ( + CallTypesLiteral, ModelResponse, ModelResponseStream, StandardLoggingPayloadErrorInformation, @@ -2008,9 +2012,10 @@ def _content_text(fields: Mapping[str, object]) -> str: class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): - self.data = data + self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place self.served_model = served_model self._tags_before_guardrails: frozenset[str] | None = None + self._rejected_before_guardrails = False self.route_type: str | None = None @property @@ -2501,14 +2506,27 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) + model_routable: Final = ( + skip_guardrails + or llm_router is None + or can_route_with_loaded_models( + data=self.data, + llm_router=llm_router, + user_model=user_model, + route_type=route_type, + ) + ) prefetch_model = self.data.get("model") if llm_router is not None and isinstance(prefetch_model, str): llm_router.arm_routing_read_prefetch(prefetch_model, self.data) - self.data = await proxy_logging_obj.pre_call_hook( + await self._run_pre_call_hooks( + proxy_logging_obj=proxy_logging_obj, user_api_key_dict=user_api_key_dict, - data=self.data, call_type=route_type, skip_guardrails=skip_guardrails, + model_routable=model_routable, + llm_router=llm_router, + user_model=user_model, ) await _enforce_guardrail_added_tag_budgets( data=self.data, @@ -2544,6 +2562,103 @@ class ProxyBaseLLMRequestProcessing: return self.data, logging_obj + async def _run_pre_call_hooks( + self, + proxy_logging_obj: ProxyLogging, + user_api_key_dict: UserAPIKeyAuth, + call_type: CallTypesLiteral, + skip_guardrails: bool, + model_routable: bool, + llm_router: Router | None, + user_model: str | None, + ) -> None: + """Run the pre-call hooks on self.data, checking a model that cannot route yet before the first guardrail. + + For such a model, the hooks that come before the first guardrail or content enforcer run + first and may rewrite the model. A model that still cannot route raises the routing error + there, before any guardrail runs, otherwise the remaining hooks and guardrails run in their + usual order. A request rejected before the check passed is redacted. + """ + if model_routable: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + skip_guardrails=skip_guardrails, + ) + return + passed_routing_check: Final = list[bool]() + + async def raise_unless_routable(request: dict[str, object]) -> None: # mutable-ok: request payload + self.data = request + try: + await raise_if_model_not_routable( + data=request, + llm_router=llm_router, + user_model=user_model, + route_type=call_type, + ) + except Exception: + refresh_proxy_server_request_body_snapshot(request) + raise + passed_routing_check.append(True) + + try: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + before_guardrails=raise_unless_routable, + ) + except Exception: + if not passed_routing_check: + self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) + raise + + def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None: + """Redact the content of a request rejected before the pre-call guardrails that apply to it ran. + + Covers every request field a pre-call guardrail can rewrite on the routes that reach this + check: chat and Anthropic messages and system, completion prompt, Responses input and + instructions, embedding and speech input, tool definitions, and Gemini contents and system + instruction. The request body, its proxy_server_request snapshot and message logging all + carry the redaction marker, so failure callbacks, post-call failure hooks and spend rows + never see content a masking guardrail would have rewritten. + """ + request_metadata: Final = dict[str, object]() + for bucket in _request_metadata_buckets(self.data): + request_metadata.update(bucket) + if not proxy_logging_obj.has_pre_call_guardrails(request_metadata, request_body=self.data): + return + self._rejected_before_guardrails = True + if "messages" in self.data: + self.data["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] + if "contents" in self.data: + self.data["contents"] = [{"role": "user", "parts": [{"text": REDACTED_BY_LITELLM}]}] + for content_key in ("prompt", "input"): + if content_key in self.data: + self.data[content_key] = "" + if "tools" in self.data: + self.data["tools"] = [] + for content_key in ("system", "instructions"): + if content_key in self.data: + self.data[content_key] = REDACTED_BY_LITELLM + redacted_system_instruction: Final = {"parts": [{"text": REDACTED_BY_LITELLM}]} + system_instruction_keys: Final = ("systemInstruction", "system_instruction") + for content_key in system_instruction_keys: + if content_key in self.data: + self.data[content_key] = redacted_system_instruction + config: Final = self.data.get("config") + if isinstance(config, Mapping): + self.data["config"] = { + **config, + **{key: redacted_system_instruction for key in system_instruction_keys if key in config}, + } + refresh_proxy_server_request_body_snapshot(self.data) + logging_obj: Final = self.data.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.standard_callback_dynamic_params["turn_off_message_logging"] = True + async def _pre_call_with_fallbacks( self, request: Request, @@ -2638,6 +2753,7 @@ class ProxyBaseLLMRequestProcessing: ): continue self.data = independent_snapshot(pristine) + self._rejected_before_guardrails = False self.data["model"] = fallback_model try: return await self.common_processing_pre_call_logic( @@ -2662,7 +2778,8 @@ class ProxyBaseLLMRequestProcessing: raise continue except BaseException: - self.data = rate_limited_data + if not self._rejected_before_guardrails: + self.data = rate_limited_data raise self.data = rate_limited_data @@ -2783,7 +2900,7 @@ class ProxyBaseLLMRequestProcessing: "Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s", len(_payload_str), MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, - (list(self.data.keys()) if isinstance(self.data, dict) else type(self.data).__name__), + sanitize_for_log(list(self.data.keys())), ) else: verbose_proxy_logger.debug( diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index cb226929167..a69e51a8e23 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -2,16 +2,18 @@ import asyncio import hashlib import json from collections import OrderedDict -from collections.abc import Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from datetime import datetime +from enum import Enum, auto from types import MappingProxyType -from typing import TYPE_CHECKING, Annotated, Any, Final, Literal +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, TypeVar from uuid import uuid4 import httpx from fastapi import HTTPException, status from pydantic import BeforeValidator, TypeAdapter, ValidationError +from typing_extensions import Never import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params @@ -44,6 +46,8 @@ if TYPE_CHECKING: else: LitellmRouter = Any +_T: Final = TypeVar("_T") + _PER_REQUEST_ROUTER_SETTINGS: Final = ( "fallbacks", @@ -721,15 +725,48 @@ async def route_request( Common helper to route the request """ router_only_settings_from_override: Final = _merge_router_settings_override(data) # pyright: ignore[reportUnknownArgumentType] # route_request takes an untyped dict body - try: - return await _route_request_single_attempt( + return await _retry_once_after_model_read_through( + data, + llm_router, + lambda router: _route_request_single_attempt( # pyright: ignore[reportUnknownLambdaType] # the router methods it returns are untyped data=data, - llm_router=llm_router, + llm_router=router, user_model=user_model, route_type=route_type, router_only_settings_from_override=router_only_settings_from_override, user_api_key_dict=user_api_key_dict, - ) + ), + ) + + +async def raise_if_model_not_routable( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, +) -> None: + """Raise the error route_request would raise before sending this request, without sending it.""" + _merge_router_settings_override(dict(data)) + + async def check(router: LitellmRouter | None) -> None: + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router) + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) + if ( + _decide_dispatch(data=data, llm_router=router, user_model=user_model, route_type=route_type) + is _Dispatch.MODEL_NOT_FOUND + ): + _raise_model_not_found(data=data, route_type=route_type) + + await _retry_once_after_model_read_through(data, llm_router, check) + + +async def _retry_once_after_model_read_through( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + attempt: Callable[[LitellmRouter | None], Awaitable[_T]], +) -> _T: + try: + return await attempt(llm_router) except (ProxyModelNotFoundError, ProxyMissingParamWithoutLoadedModelError) as e: requested_model: Final = data.get("model", "") if ( @@ -745,73 +782,67 @@ async def route_request( if not await model_registry_read_through.attempt(requested_model): raise - return await _route_request_single_attempt( - data=data, - llm_router=proxy_server.llm_router, - user_model=user_model, - route_type=route_type, - router_only_settings_from_override=router_only_settings_from_override, - user_api_key_dict=user_api_key_dict, - ) + return await attempt(proxy_server.llm_router) -async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed - data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request +def _raise_model_not_found(data: Mapping[str, object], route_type: str) -> Never: + # if no route found then it's a bad request + route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) + raise ProxyModelNotFoundError( + route=route_name, + model_name=str(data.get("model", "")), + ) + + +class _Dispatch(Enum): + ROUTER = auto() + ROUTER_SPECIFIC_DEPLOYMENT = auto() + ROUTER_THEN_LITELLM = auto() + LITELLM = auto() + BATCH_COMPLETION = auto() + USER_CONFIG = auto() + EVAL = auto() + A2A_AGENT = auto() + MISSING_MODEL = auto() + MODEL_NOT_FOUND = auto() + + +@dataclass(frozen=True, slots=True) +class _TeamModel: + name: str + + +def _decide_dispatch( # noqa: C901 # one branch per dispatch target, moved out of _route_request_single_attempt + data: Mapping[str, object], llm_router: LitellmRouter | None, user_model: str | None, - route_type: RouteType, - router_only_settings_from_override: frozenset[str], - user_api_key_dict: UserAPIKeyAuth | None = None, -): - raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router) + route_type: str, +) -> _Dispatch | _TeamModel: + """Where route_request sends this request, decided without changing it. - await add_shared_session_to_data(data) - - raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) - - def direct_litellm_kwargs() -> Mapping[str, object]: - return MappingProxyType( - { - key: value - for key, value in JSON_OBJECT_ADAPTER.validate_python(data).items() - if key not in router_only_settings_from_override - } - ) - - team_id: Final = get_team_id_from_data(data) + Raises PermissionDeniedError (403) when every deployment of the requested model is blocked. + """ + model: Final = data.get("model") + team_id: Final = get_team_id_from_data(dict(data)) router_model_names: Final = llm_router.model_names if llm_router is not None else [] is_proxy_admin_without_team: Final = team_id is None and is_proxy_admin_request(data) - # Preprocess Google GenAI generate content requests - if route_type in ["agenerate_content", "agenerate_content_stream"]: - # Map generationConfig to config parameter for Google GenAI compatibility - if "generationConfig" in data and "config" not in data: - data["config"] = data.pop("generationConfig") if "api_key" in data or "api_base" in data: if llm_router is not None: - return getattr(llm_router, f"{route_type}")(**data) + return _Dispatch.ROUTER else: - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + return _Dispatch.LITELLM - elif ( - route_type == "acompletion" - and data.get("model", "") is not None - and "," in data.get("model", "") - and llm_router is not None - ): + elif route_type == "acompletion" and isinstance(model, str) and "," in model and llm_router is not None: # Handle batch completions with comma-separated models BEFORE user_config check # This ensures batch completion logic is applied even when user_config is set - if data.get("fastest_response", False): - return llm_router.abatch_completion_fastest_response(**data) - else: - models: Final = [model.strip() for model in data.pop("model").split(",")] - return llm_router.abatch_completion(models=models, **data) + return _Dispatch.BATCH_COMPLETION - elif "user_config" in data: - return _route_user_config_request(data, route_type) + elif data.get("user_config") is not None: + return _Dispatch.USER_CONFIG elif llm_router is not None: - _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) + _raise_if_model_fully_blocked(llm_router=llm_router, model_name=model, team_id=team_id) # Evals API: always route to litellm directly (not through router) # But extract model credentials if a model is provided if route_type in [ @@ -827,30 +858,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "acancel_run", "adelete_run", ]: - # If a model is provided, get its credentials from the router - model: Final = data.get("model") - if model and llm_router: - try: - # Try to get deployment credentials for this model - deployment_creds = llm_router.get_deployment_credentials(model_id=model) - if not deployment_creds: - # Try by model group name - deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) - if ( - deployment - and deployment.litellm_params - and not llm_router.is_deployment_blocked(deployment) - ): - deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) - - # If we found credentials, merge them into data (but don't override user-provided values) - if deployment_creds: - data.update(deployment_creds) - except Exception: - # If we can't get deployment creds, continue without them - pass - - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + return _Dispatch.EVAL # Skip model-based routing for container operations if route_type in [ "acreate_container", @@ -863,7 +871,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "adelete_container_file", "aretrieve_container_file_content", ]: - return getattr(llm_router, f"{route_type}")(**data) + return _Dispatch.ROUTER # Interactions API: create with agent, get/delete/cancel don't need model routing if route_type in [ "acreate_interaction", @@ -871,7 +879,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "adelete_interaction", "acancel_interaction", ]: - return getattr(llm_router, f"{route_type}")(**data) + return _Dispatch.ROUTER # Managed Agents API: these don't need model routing if route_type in [ "acreate_agent", @@ -880,7 +888,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "adelete_agent", "alist_agent_versions", ]: - return getattr(llm_router, f"{route_type}")(**data) + return _Dispatch.ROUTER if route_type in [ "avideo_list", "avideo_status", @@ -899,17 +907,14 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "aget_skill", "adelete_skill", "aingest", - ] and (data.get("model") is None or data.get("model") == ""): + ] and (model is None or model == ""): # These endpoints don't need a model, use custom_llm_provider directly - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + return _Dispatch.LITELLM if "model" not in data: - raise ProxyMissingRequiredParamError( - route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), - param="model", - ) - custom_llm_provider: Final[object] = ( - JSON_OBJECT_ADAPTER.validate_python(data).get("custom_llm_provider") + return _Dispatch.MISSING_MODEL + custom_llm_provider: Final = ( + data.get("custom_llm_provider") if route_type in ( "avector_store_search", @@ -921,31 +926,33 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr ) team_model_provider: Final[str | None] = custom_llm_provider if isinstance(custom_llm_provider, str) else None team_model_name: Final = ( - llm_router.map_team_model(data["model"], team_id, custom_llm_provider=team_model_provider) - if team_id is not None + llm_router.map_team_model(model, team_id, custom_llm_provider=team_model_provider) + if team_id is not None and (model is None or isinstance(model, str)) else None ) if team_model_name is not None: - data["model"] = team_model_name - return getattr(llm_router, f"{route_type}")(**data) + return _TeamModel(name=team_model_name) - elif ( - is_proxy_admin_without_team - and data["model"] not in router_model_names - and data["model"] in llm_router.team_public_model_names - ) or llm_router.is_recognized_model(data["model"]): - return getattr(llm_router, f"{route_type}")(**data) + elif isinstance(model, str) and ( + ( + is_proxy_admin_without_team + and model not in router_model_names + and model in llm_router.team_public_model_names + ) + or llm_router.is_recognized_model(model) + ): + return _Dispatch.ROUTER - elif data["model"] not in router_model_names: + elif model not in router_model_names: # Check wildcards before checking deployment_names # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match if llm_router.router_general_settings.pass_through_all_models: - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + return _Dispatch.LITELLM elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: - return getattr(llm_router, f"{route_type}")(**data) - elif data["model"] in llm_router.deployment_names: + return _Dispatch.ROUTER + elif model in llm_router.deployment_names: # Only match deployment_names if no wildcard matched - return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True) + return _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT elif route_type in [ "amoderation", "aget_responses", @@ -976,7 +983,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "aretrieve_container_file_content", ]: # These endpoints can work with or without model parameter - return getattr(llm_router, f"{route_type}")(**data) + return _Dispatch.ROUTER elif route_type in [ "avideo_status", "avideo_content", @@ -988,27 +995,126 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr ]: # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), # try router first to allow for multi-deployment load balancing - try: - return getattr(llm_router, f"{route_type}")(**data) - except Exception: - # If router fails (e.g., model not found in router), fall back to direct call - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) - elif _is_a2a_agent_model(data.get("model", "")): - from litellm.proxy.agent_endpoints.a2a_routing import ( - route_a2a_agent_request, - ) - - result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict) - if result is not None: - return result - # Fall through to raise exception below if result is None + return _Dispatch.ROUTER_THEN_LITELLM + elif _is_a2a_agent_model(model): + return _Dispatch.A2A_AGENT elif user_model is not None or route_type == "allm_passthrough_route": - return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + return _Dispatch.LITELLM - # if no route found then it's a bad request - route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) - raise ProxyModelNotFoundError( - route=route_name, - model_name=data.get("model", ""), - ) + return _Dispatch.MODEL_NOT_FOUND + + +def can_route_with_loaded_models( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, +) -> bool: + """Whether route_request would send this request somewhere, using only the models this worker has loaded. + + A fully blocked model counts as not routable. + """ + try: + return ( + _decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type) + is not _Dispatch.MODEL_NOT_FOUND + ) + except litellm.PermissionDeniedError: + return False + + +async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed + data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: RouteType, + router_only_settings_from_override: frozenset[str], + user_api_key_dict: UserAPIKeyAuth | None = None, +): + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router) + + await add_shared_session_to_data(data) + + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) + + def direct_litellm_kwargs() -> Mapping[str, object]: + return MappingProxyType( + { + key: value + for key, value in JSON_OBJECT_ADAPTER.validate_python(data).items() + if key not in router_only_settings_from_override + } + ) + + # Preprocess Google GenAI generate content requests + if route_type in ["agenerate_content", "agenerate_content_stream"]: + # Map generationConfig to config parameter for Google GenAI compatibility + if "generationConfig" in data and "config" not in data: + data["config"] = data.pop("generationConfig") + + match _decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): + case _Dispatch.ROUTER: + return getattr(llm_router, f"{route_type}")(**data) + case _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT: + return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True) + case _Dispatch.ROUTER_THEN_LITELLM: + try: + return getattr(llm_router, f"{route_type}")(**data) + except Exception: + # If router fails (e.g., model not found in router), fall back to direct call + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + case _Dispatch.LITELLM: + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + case _Dispatch.BATCH_COMPLETION if llm_router is not None: + if data.get("fastest_response", False): + return llm_router.abatch_completion_fastest_response(**data) + models: Final = [model.strip() for model in data.pop("model").split(",")] + return llm_router.abatch_completion(models=models, **data) + case _Dispatch.USER_CONFIG: + return _route_user_config_request(data, route_type) + case _Dispatch.EVAL: + # If a model is provided, get its credentials from the router + model: Final = data.get("model") + if model and llm_router: + try: + # Try to get deployment credentials for this model + deployment_creds = llm_router.get_deployment_credentials(model_id=model) + if not deployment_creds: + # Try by model group name + deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) + if ( + deployment + and deployment.litellm_params + and not llm_router.is_deployment_blocked(deployment) + ): + deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) + + # If we found credentials, merge them into data (but don't override user-provided values) + if deployment_creds: + data.update(deployment_creds) + except Exception: + # If we can't get deployment creds, continue without them + pass + + return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs()) + case _TeamModel(name=team_model_name): + data["model"] = team_model_name + return getattr(llm_router, f"{route_type}")(**data) + case _Dispatch.MISSING_MODEL: + raise ProxyMissingRequiredParamError( + route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), + param="model", + ) + case _Dispatch.A2A_AGENT: + from litellm.proxy.agent_endpoints.a2a_routing import ( + route_a2a_agent_request, + ) + + result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict) + if result is not None: + return result + case _Dispatch.BATCH_COMPLETION | _Dispatch.MODEL_NOT_FOUND: + pass + + _raise_model_not_found(data=data, route_type=route_type) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0ddfc5ce62f..9bd39132bc3 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2867,6 +2867,7 @@ class ProxyLogging: def has_pre_call_guardrails( self, request_metadata: Mapping[str, object], + request_body: Mapping[str, object] | None = None, event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, ) -> bool: """ @@ -2875,17 +2876,19 @@ class ProxyLogging: Evaluated with the same predicate the pre-call loop uses, so a proxy configured only with post-call guardrails answers False. Callers that must pay a real cost to build the hook's input, such as streaming a batch input file off disk, use this to skip that work. + ``request_body`` supplies the request's top-level fields, such as ``stream``, that a + guardrail scoped to streaming or non-streaming requests reads. A content-enforcing ``CustomLogger`` counts too. It is not a guardrail and has no event hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + probe: Final = {**(request_body or {}), "metadata": dict(request_metadata)} + if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(probe)): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} return any( isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type) for callback in caps.resolved_callbacks @@ -2901,6 +2904,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> None: pass @@ -2913,6 +2917,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> dict: pass @@ -2924,6 +2929,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> dict | None: """ Allows users to modify/reject the incoming request to the proxy, without having to deal with parsing Request body. @@ -2936,6 +2942,10 @@ class ProxyLogging: With ``guardrails_only`` the walk is limited to guardrails and guardrail pipelines: rate limiting, budget accounting, prompt templates and hanging-request alerting are skipped. Use it to scan a payload that is not itself a request, such as one record of a batch file. + + ``before_guardrails`` is awaited once with the request, right before the first guardrail + pipeline, guardrail or content enforcer that runs on it, or after the other callbacks when + none does. An exception it raises ends the hook. """ verbose_proxy_logger.debug("Inside Proxy Logging Pre-call hook!") @@ -2986,9 +2996,16 @@ class ProxyLogging: raw_request_snapshot: Final[dict | None] = ( # mutable-ok: same request-payload shape as data independent_snapshot(data) if needs_raw_request_snapshot else None ) + pending_before_guardrails: Final = [before_guardrails] if before_guardrails is not None else [] + + async def reach_guardrails(request: dict[str, object]) -> None: # mutable-ok: request payload + while pending_before_guardrails: + await pending_before_guardrails.pop()(request) try: if not skip_guardrails: + if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(data)): + await reach_guardrails(data) data, _ = await self._maybe_execute_pipelines( # rebind-ok: pipeline edits feed the callback loop below data=data, user_api_key_dict=user_api_key_dict, @@ -3014,6 +3031,7 @@ class ProxyLogging: and (guardrails_only or not caps.has_pre_call_override) ): if data is not None: + await reach_guardrails(data) self._process_guardrail_metadata(data) return data @@ -3044,6 +3062,13 @@ class ProxyLogging: if getattr(_callback, "run_in_parallel", False): continue + if ( + pending_before_guardrails + and _callback.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call) + is True + ): + await reach_guardrails(data) + data = await self._run_sequential_guardrail_callback( callback=_callback, data=data, @@ -3062,6 +3087,9 @@ class ProxyLogging: if call_type in MCP_GUARDRAIL_CALL_TYPES and user_api_key_dict is None: continue + if _callback.enforces_request_content and data is not None: + await reach_guardrails(data) + response: Exception | str | Mapping[str, object] | None = ( await _callback.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -3105,6 +3133,9 @@ class ProxyLogging: end_time=end_time, ) + if data is not None: + await reach_guardrails(data) + if deferred_route_exc is not None and data is not None: data = await self._handle_sensitive_data_route_exception(deferred_route_exc, data, user_api_key_dict) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 52816dc50a8..40c0a1efd84 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -3,7 +3,7 @@ import contextvars import copy import datetime import json -from collections.abc import AsyncIterable, Mapping +from collections.abc import AsyncIterable, Awaitable, Mapping from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus @@ -33,6 +33,7 @@ from litellm.constants import ( RETURN_RAW_MODEL_NAME_METADATA_KEY, STREAM_SSE_KEEPALIVE_PING_BYTES, ) +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( @@ -80,6 +81,7 @@ from litellm.router import Router from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.utils import ModelResponseListIterator from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment +from litellm.types.guardrails import GuardrailEventHooks def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): @@ -11901,6 +11903,610 @@ class TestStreamingContainerOwnershipRecordedBeforeDone: recorder.assert_awaited_once() +class _CountingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="counting-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True) + self.models_seen: list[object] = [] + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.models_seen.append(data.get("model")) + return data + + +class _CountingContentEnforcer(CustomLogger): + enforces_request_content = True + + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + return data + + +class _ModelRewritingHook(CustomLogger): + def __init__(self, source_model: str = "virtual-model", target_model: str = "gpt-4o-mini") -> None: + super().__init__() + self.source_model = source_model + self.target_model = target_model + self.calls = 0 + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + if data.get("model") == self.source_model: + data["model"] = self.target_model + return data + + +class _ModelReplacingHook(CustomLogger): + def __init__(self, target_model: str) -> None: + super().__init__() + self.target_model = target_model + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + return {**data, "model": self.target_model} + + +class _EmailMaskingHook(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + data["messages"] = [ + {**message, "content": message["content"].replace("jane@example.com", "[EMAIL]")} + for message in data["messages"] + ] + return data + + +class _EmailMaskingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__( + guardrail_name="email-masking-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + for message in data["messages"]: + message["content"] = message["content"].replace("jane@example.com", "[EMAIL]") + return data + + +class _ContentRecordingHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.contents_seen: list[str] = [] + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.contents_seen.extend(message["content"] for message in data["messages"]) + return data + + +class _RejectingHook(CustomLogger): + def __init__(self, model: str | None = None) -> None: + super().__init__() + self.model = model + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + if self.model in (None, data.get("model")): + raise HTTPException(status_code=403, detail="rejected by hook") + return data + + +class _RateLimitedModelHook(CustomLogger): + def __init__(self, model: str) -> None: + super().__init__() + self.model = model + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + if data.get("model") == self.model: + raise ProxyRateLimitError(detail=f"rpm limit reached for {self.model}") + return data + + +class TestPreCallHooksForUnroutableModel: + @staticmethod + async def _run( + monkeypatch, + model: str, + llm_router: Router | None, + processor: ProxyBaseLLMRequestProcessing | None = None, + skip_guardrails: bool = False, + model_db_lookup: Callable[[str], Awaitable[bool]] | None = None, + extra_body: Mapping[str, object] | None = None, + ) -> dict: + from litellm.proxy import proxy_server + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + registry_read_through.RegistryReadThrough( + resync=model_db_lookup or registry_read_through._resync_model_deployments, + is_loaded=registry_read_through._model_is_loaded, + ), + ) + target: Final = processor or ProxyBaseLLMRequestProcessing(data={}) + target.data.update( + { + "model": model, + "messages": [{"role": "user", "content": "email jane@example.com"}], + **copy.deepcopy(dict(extra_body or {})), + } + ) + request: Final = Request( + {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""} + ) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + data, _ = await target.common_processing_pre_call_logic( + request=request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=proxy_logging_obj, + proxy_config=proxy_server.ProxyConfig(), + route_type="acompletion", + llm_router=llm_router, + skip_guardrails=skip_guardrails, + ) + return data + + @staticmethod + def _router() -> Router: + return Router( + model_list=[ + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + { + "model_name": "blocked-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"blocked": True}, + }, + ] + ) + + @pytest.mark.asyncio + async def test_unknown_model_runs_only_the_hooks_before_the_first_guardrail(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router()) + + assert exc_info.value.status_code == 400 + assert hook.calls == 1 + assert guardrail.models_seen == [] + assert enforcer.calls == 0 + + @pytest.mark.asyncio + async def test_hook_that_rewrites_unknown_model_to_routable_one_still_gets_guardrails(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert hook.calls == 1 + assert guardrail.models_seen == ["gpt-4o-mini"] + assert enforcer.calls == 1 + + @pytest.mark.asyncio + async def test_unknown_model_never_reaches_a_hook_registered_after_a_guardrail(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _EmailMaskingGuardrail() + recorder: Final = _ContentRecordingHook() + rejecting_hook: Final = _RejectingHook() + monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder, rejecting_hook]) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router()) + + assert exc_info.value.status_code == 400 + assert recorder.contents_seen == [] + + @pytest.mark.asyncio + async def test_hook_registered_after_a_guardrail_sees_the_guardrail_output_for_a_renamed_model(self, monkeypatch): + recorder: Final = _ContentRecordingHook() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _EmailMaskingGuardrail(), recorder]) + + data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert recorder.contents_seen == ["email [EMAIL]"] + + @pytest.mark.asyncio + async def test_routable_model_runs_every_hook_once(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router()) + + assert (hook.calls, guardrail.models_seen, enforcer.calls) == (1, ["gpt-4o-mini"], 1) + + @pytest.mark.asyncio + async def test_routable_model_keeps_the_configured_hook_order(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail, _RejectingHook()]) + + with pytest.raises(HTTPException): + await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router()) + + assert guardrail.models_seen == ["gpt-4o-mini"] + + @pytest.mark.asyncio + async def test_fully_blocked_model_is_a_403_without_guardrails(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + with pytest.raises(litellm.PermissionDeniedError): + await self._run(monkeypatch, model="blocked-model", llm_router=self._router()) + + assert guardrail.models_seen == [] + + @pytest.mark.asyncio + async def test_hook_that_rewrites_blocked_model_to_routable_one_is_served(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(source_model="blocked-model"), guardrail]) + + data: Final = await self._run(monkeypatch, model="blocked-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert guardrail.models_seen == ["gpt-4o-mini"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("callbacks", "expected"), + [ + pytest.param(lambda: [_CountingGuardrail()], True, id="guardrail-applies"), + pytest.param(lambda: [_CountingContentEnforcer()], True, id="content-enforcer"), + pytest.param(lambda: [_ModelRewritingHook()], None, id="no-guardrail"), + ], + ) + async def test_rejected_request_is_redacted_only_when_guardrails_were_bypassed( + self, monkeypatch, callbacks, expected + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", callbacks()) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + ) + + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is expected + assert logging_obj.model_call_details["standard_callback_dynamic_params"] is ( + logging_obj.standard_callback_dynamic_params + ) + expected_content: Final = "redacted-by-litellm" if expected else "email jane@example.com" + assert processor.data["messages"] == [{"role": "user", "content": expected_content}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + + @pytest.mark.asyncio + async def test_request_a_hook_rejects_before_guardrails_is_redacted(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_RejectingHook(), guardrail]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor) + + assert exc_info.value.status_code == 403 + assert guardrail.models_seen == [] + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is True + + @pytest.mark.asyncio + async def test_request_rejected_after_the_routing_check_passed_is_not_redacted(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), guardrail, _RejectingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException) as exc_info: + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + assert exc_info.value.status_code == 403 + assert guardrail.models_seen == ["gpt-4o-mini"] + assert processor.data["messages"] == [{"role": "user", "content": "email jane@example.com"}] + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is not True + + @pytest.mark.asyncio + async def test_request_rejected_after_the_routing_check_passed_logs_the_request_as_sent(self, monkeypatch): + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _CountingGuardrail(), _RejectingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException): + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert processor.data["model"] == "gpt-4o-mini" + assert (snapshot["model"], snapshot["messages"], "litellm_call_id" in snapshot) == ( + "virtual-model", + [{"role": "user", "content": "email jane@example.com"}], + False, + ), snapshot + + @pytest.mark.asyncio + async def test_request_a_hook_replaced_with_another_unknown_model_is_the_one_rejected_and_redacted( + self, monkeypatch + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_ModelReplacingHook("still-unknown"), _CountingGuardrail()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + assert processor.data["model"] == "still-unknown" + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("extra_body", "expected_body"), + [ + pytest.param( + {"system": [{"type": "text", "text": "email jane@example.com"}]}, + {"system": "redacted-by-litellm"}, + id="anthropic-system", + ), + pytest.param( + {"instructions": "email jane@example.com"}, + {"instructions": "redacted-by-litellm"}, + id="responses-instructions", + ), + pytest.param( + {"contents": [{"role": "user", "parts": [{"text": "email jane@example.com"}]}]}, + {"contents": [{"role": "user", "parts": [{"text": "redacted-by-litellm"}]}]}, + id="gemini-contents", + ), + pytest.param( + {"systemInstruction": {"parts": [{"text": "email jane@example.com"}]}}, + {"systemInstruction": {"parts": [{"text": "redacted-by-litellm"}]}}, + id="gemini-system-instruction", + ), + pytest.param( + {"config": {"system_instruction": {"parts": [{"text": "email jane@example.com"}]}, "temperature": 0}}, + {"config": {"system_instruction": {"parts": [{"text": "redacted-by-litellm"}]}, "temperature": 0}}, + id="gemini-config-system-instruction", + ), + pytest.param( + {"prompt": "email jane@example.com", "input": ["email jane@example.com"], "max_tokens": 5}, + {"prompt": "", "input": "", "max_tokens": 5}, + id="prompt-and-input", + ), + pytest.param( + {"tools": [{"type": "function", "function": {"name": "mail", "description": "jane@example.com"}}]}, + {"tools": []}, + id="tool-definitions", + ), + ], + ) + async def test_rejected_request_redacts_every_guardrail_scanned_field(self, monkeypatch, extra_body, expected_body): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + extra_body=extra_body, + ) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert {key: processor.data[key] for key in expected_body} == expected_body + assert {key: snapshot[key] for key in expected_body} == expected_body + + @pytest.mark.asyncio + async def test_unknown_model_hooks_left_unchanged_gets_one_registry_read_through(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()]) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), model_db_lookup=no_db_row) + + assert looked_up == ["does-not-exist"] + + @pytest.mark.asyncio + async def test_model_rewritten_by_a_hook_is_looked_up_once_under_its_new_name(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr( + litellm, + "callbacks", + [_ModelRewritingHook(source_model="unknown-virtual", target_model="db-only-model")], + ) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), model_db_lookup=no_db_row) + + assert looked_up == ["db-only-model"] + + @pytest.mark.asyncio + async def test_virtual_model_a_hook_maps_to_a_loaded_model_needs_no_db_lookup(self, monkeypatch): + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook()]) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False + + data: Final = await self._run( + monkeypatch, model="virtual-model", llm_router=self._router(), model_db_lookup=no_db_row + ) + + assert data["model"] == "gpt-4o-mini" + assert looked_up == [] + + @pytest.mark.asyncio + async def test_unknown_model_without_guardrails_logs_the_request_the_hooks_produced(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_EmailMaskingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert snapshot["messages"] == [{"role": "user", "content": "email [EMAIL]"}] + assert snapshot["litellm_call_id"] == processor.data["litellm_call_id"] + assert snapshot["metadata"] is processor.data["metadata"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("stream_scope", "stream", "expected_content"), + [ + pytest.param("streaming", True, "redacted-by-litellm", id="streaming-guardrail-streaming-request"), + pytest.param("streaming", False, "email jane@example.com", id="streaming-guardrail-plain-request"), + pytest.param("non_streaming", True, "email jane@example.com", id="plain-guardrail-streaming-request"), + ], + ) + async def test_rejected_request_is_redacted_when_a_guardrail_scoped_to_its_stream_mode_applies( + self, monkeypatch, stream_scope, stream, expected_content + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _CountingGuardrail() + guardrail.apply_stream_scope(stream_scope) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + extra_body={"stream": stream}, + ) + + assert processor.data["messages"] == [{"role": "user", "content": expected_content}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + + @pytest.mark.asyncio + async def test_caller_that_skips_guardrails_runs_content_enforcers_and_no_guardrail_for_a_renamed_model( + self, monkeypatch + ): + enforcer: Final = _CountingContentEnforcer() + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), enforcer, guardrail]) + + data: Final = await self._run( + monkeypatch, model="virtual-model", llm_router=self._router(), skip_guardrails=True + ) + + assert data["model"] == "gpt-4o-mini" + assert enforcer.calls == 1 + assert guardrail.models_seen == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("primary", "fallback", "rejecting_hook", "expected_status"), + [ + pytest.param("gpt-4o-mini", "does-not-exist", "unused-model", 400, id="fallback-model-unknown"), + pytest.param("does-not-exist", "gpt-4o-mini", "gpt-4o-mini", 403, id="primary-model-unknown"), + ], + ) + async def test_rate_limit_fallback_logs_the_attempt_rejected_before_its_guardrails( + self, monkeypatch, primary, fallback, rejecting_hook, expected_status + ): + from litellm.proxy import proxy_server + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + registry_read_through.RegistryReadThrough( + resync=registry_read_through._resync_model_deployments, + is_loaded=registry_read_through._model_is_loaded, + ), + ) + monkeypatch.setattr( + litellm, + "callbacks", + [_RateLimitedModelHook(model=primary), _RejectingHook(model=rejecting_hook), _CountingGuardrail()], + ) + router: Final = Router( + model_list=[ + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}} + ], + fallbacks=[{primary: [fallback]}], + ) + processor: Final = ProxyBaseLLMRequestProcessing( + data={"model": primary, "messages": [{"role": "user", "content": "email jane@example.com"}]} + ) + + request: Final = Request( + {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""} + ) + + with pytest.raises((ProxyModelNotFoundError, HTTPException)) as exc_info: + await processor._pre_call_with_fallbacks( + request=request, + general_settings={}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + user_api_key_dict=ProxyUserAPIKeyAuth(), + version=None, + proxy_config=proxy_server.ProxyConfig(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + route_type="acompletion", + llm_router=router, + ) + + assert exc_info.value.status_code == expected_status + assert processor.data["model"] == "does-not-exist" + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + + @pytest.mark.asyncio + async def test_no_router_skips_the_routability_check(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + data: Final = await self._run(monkeypatch, model="router-alias-only-known-at-dispatch", llm_router=None) + + assert data["model"] == "router-alias-only-known-at-dispatch" + assert guardrail.models_seen == ["router-alias-only-known-at-dispatch"] + + class TestResponseCostFromLoggingObjForBridgedMessages: """A /v1/messages request answered through the chat bridge arrives here as an Anthropic body whose usage has no reasoning field, while the cost it recomputes must still price the reasoning tokens diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 6044475652c..99036db38d3 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -26,6 +26,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.route_llm_request import ( JSON_OBJECT_ADAPTER, + MockTestingParamsDisabledError, + ProxyMissingRequiredParamError, ProxyModelNotFoundError, UserConfigRouters, route_request, @@ -2186,6 +2188,184 @@ def test_proxy_model_not_found_error_keeps_the_raw_model_only_in_the_client_resp assert error.spend_log_error_message.startswith("/chat/completions: Invalid model name passed in") +def _router_with_gpt4o_mini(router_general_settings: RouterGeneralSettings | None = None): + import litellm + + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + } + ], + router_general_settings=router_general_settings, + ) + + +@pytest.mark.parametrize( + "extra_body", + [ + pytest.param({}, id="plain"), + pytest.param({"user_config": None}, id="null-user-config"), + pytest.param({"router_settings_override": {"num_retries": 3}}, id="key-router-settings"), + ], +) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_raises_for_unknown_model(extra_body): + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await raise_if_model_not_routable( + data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], **extra_body}, + llm_router=_router_with_gpt4o_mini(), + user_model=None, + route_type="acompletion", + ) + + assert exc_info.value.status_code == 400 + assert "does-not-exist" in str(exc_info.value.detail) + + +@pytest.mark.parametrize( + ("data", "route_type", "expected_error"), + [ + pytest.param( + {"model": "does-not-exist"}, + "atext_completion", + ProxyMissingRequiredParamError, + id="missing-prompt", + ), + pytest.param( + {"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], "mock_timeout": True}, + "acompletion", + MockTestingParamsDisabledError, + id="mock-params-disallowed", + ), + ], +) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_reports_body_errors_before_the_model(data, route_type, expected_error): + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + with pytest.raises(expected_error): + await raise_if_model_not_routable( + data=data, + llm_router=_router_with_gpt4o_mini(), + user_model=None, + route_type=route_type, + ) + + +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_reports_a_malformed_router_settings_override_like_route_request(): + from litellm.proxy._types import ProxyException + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + data: Final = { + "model": "does-not-exist", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": "not-an-object", + } + + with pytest.raises(ProxyException) as from_check: + await raise_if_model_not_routable( + data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion" + ) + with pytest.raises(ProxyException) as from_route: + await route_request(dict(data), _router_with_gpt4o_mini(), None, "acompletion") + + assert from_check.value.param == from_route.value.param == "router_settings_override" + assert from_check.value.message == from_route.value.message + assert data["router_settings_override"] == "not-an-object" + + +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_leaves_the_router_settings_override_for_route_request(): + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + data: Final = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": {"num_retries": 3}, + } + + await raise_if_model_not_routable( + data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion" + ) + + assert data == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": {"num_retries": 3}, + } + + +@pytest.mark.parametrize( + "data, route_type, router_mutator", + [ + ( + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "openai/anything", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + "wildcard", + ), + ( + {"model": "does-not-exist", "api_key": "sk-user", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "does-not-exist,gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + "pass_through_all_models", + ), + ( + {"model": "does-not-exist", "input": "hello"}, + "amoderation", + None, + ), + ], +) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_allows_routable_requests(data, route_type, router_mutator): + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + if router_mutator == "wildcard": + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-test"}, + } + ] + ) + elif router_mutator == "pass_through_all_models": + router = _router_with_gpt4o_mini(router_general_settings=RouterGeneralSettings(pass_through_all_models=True)) + else: + router = _router_with_gpt4o_mini() + + assert ( + await raise_if_model_not_routable( + data=data, + llm_router=router, + user_model=None, + route_type=route_type, + ) + is None + ) + + @pytest.mark.asyncio async def test_route_request_without_model_on_model_routed_endpoint_is_a_400(): import litellm @@ -2370,6 +2550,72 @@ async def test_null_user_config_is_treated_as_absent() -> None: assert response.choices[0].message.content == "from the main router" +@pytest.mark.asyncio +async def test_route_request_without_a_model_sends_to_the_team_deployment_by_its_public_name(): + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "internal-team-gpt", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "from-team"}, + "model_info": {"id": "team-gpt-1", "team_id": "team-a", "team_public_model_name": "team-gpt"}, + } + ] + ) + data = { + "model": None, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_api_key_team_id": "team-a"}, + } + + llm_call = await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion") + response = await llm_call + + assert response.choices[0].message.content == "from-team" + assert data["model"] == "team-gpt" + + +@pytest.mark.asyncio +async def test_route_request_read_through_sends_through_the_router_it_built(monkeypatch): + """With no router loaded yet, a model the read-through finds is sent through the router the read-through built.""" + import litellm + from litellm.proxy import proxy_server + + model_name = "e2e-first-db-model" + built_router = litellm.Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "fake", + "mock_response": "hello-from-new-router", + }, + } + ] + ) + fake_prisma, _ = _fake_prisma_client_with_models([_db_model_row(model_name, "unused")]) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", None) + + async def build_router_from_db(prisma_client, proxy_logging_obj): + monkeypatch.setattr(proxy_server, "llm_router", built_router) + + monkeypatch.setattr(proxy_server.proxy_config, "add_deployment", build_router_from_db) + + llm_call = await route_request( + data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]}, + llm_router=None, + user_model=None, + route_type="acompletion", + ) + response = await llm_call + + assert response.choices[0].message.content == "hello-from-new-router" + + def _registered_bedrock_kb_search(team_id: str) -> dict[str, object]: """The request a team key's /v1/vector_stores/{id}/search carries once the registry row is merged in.""" return { diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index b7fc8c6dbeb..649598ad8a8 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -266,10 +266,14 @@ def test_has_pre_call_guardrails_is_false_without_callbacks(proxy_logging, monke assert proxy_logging.has_pre_call_guardrails({}) is False -def test_has_pre_call_guardrails_is_true_for_a_configured_pipeline(proxy_logging, monkeypatch): - monkeypatch.setattr(litellm, "callbacks", []) +@pytest.mark.parametrize(("mode", "expected"), [("pre_call", True), ("post_call", False)]) +def test_has_pre_call_guardrails_follows_the_pipeline_mode(proxy_logging, monkeypatch, mode, expected): + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep - assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": ["p1"]}) is True + monkeypatch.setattr(litellm, "callbacks", []) + pipeline: Final = GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="g")]) + + assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": [("policy", pipeline)]}) is expected @pytest.mark.asyncio @@ -995,3 +999,134 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +class _NamedHook(CustomLogger): + def __init__(self, name: str, seen: list[str]) -> None: + super().__init__() + self.name = name + self.seen = seen + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.name) + return data + + +class _NamedEnforcer(_NamedHook): + enforces_request_content = True + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.name) + return data + + +class _NamedGuardrail(CustomGuardrail): + def __init__(self, name: str, seen: list[str], default_on: bool = True, run_in_parallel: bool = False) -> None: + super().__init__(guardrail_name=name, event_hook=GuardrailEventHooks.pre_call, default_on=default_on) + self.seen = seen + self.run_in_parallel = run_in_parallel + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.guardrail_name) + return data + + +def _callbacks(spec: str, seen: list[str]) -> list[CustomLogger]: + def build(name: str) -> CustomLogger: + if name.startswith("enforcer"): + return _NamedEnforcer(name, seen) + if name.startswith("off-guardrail"): + return _NamedGuardrail(name, seen, default_on=False) + if name.startswith("parallel-guardrail"): + return _NamedGuardrail(name, seen, run_in_parallel=True) + if name.startswith("guardrail"): + return _NamedGuardrail(name, seen) + return _NamedHook(name, seen) + + return [build(name) for name in spec.split()] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("callbacks", "expected"), + [ + pytest.param("hook-a guardrail hook-b", ["hook-a", "check", "guardrail", "hook-b"], id="guardrail"), + pytest.param("hook-a enforcer hook-b", ["hook-a", "check", "enforcer", "hook-b"], id="content-enforcer"), + pytest.param( + "off-guardrail hook-a guardrail hook-b", + ["hook-a", "check", "guardrail", "hook-b"], + id="guardrail-that-does-not-apply", + ), + pytest.param( + "hook-a parallel-guardrail hook-b", + ["hook-a", "hook-b", "check", "parallel-guardrail"], + id="parallel-guardrail", + ), + pytest.param("hook-a hook-b", ["hook-a", "hook-b", "check"], id="no-guardrail"), + pytest.param("", ["check"], id="no-callbacks"), + ], +) +async def test_before_guardrails_runs_once_right_before_the_first_guardrail( + proxy_logging, make_user_api_key_auth, monkeypatch, callbacks, expected +): + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks(callbacks, seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + + async def check(request: dict) -> None: + seen.append("check") + + await proxy_logging.pre_call_hook( + user_api_key_dict=make_user_api_key_auth(), + data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + call_type="acompletion", + before_guardrails=check, + ) + + assert seen == expected + + +@pytest.mark.asyncio +async def test_before_guardrails_runs_before_a_guardrail_pipeline(proxy_logging, make_user_api_key_auth, monkeypatch): + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail-in-pipeline", seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + pipeline: Final = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="guardrail-in-pipeline")]) + + async def check(request: dict) -> None: + seen.append("check") + + await proxy_logging.pre_call_hook( + user_api_key_dict=make_user_api_key_auth(), + data={ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("policy", pipeline)]}, + }, + call_type="acompletion", + before_guardrails=check, + ) + + assert seen == ["check", "guardrail-in-pipeline", "hook-a"] + + +@pytest.mark.asyncio +async def test_a_failing_before_guardrails_stops_the_walk(proxy_logging, make_user_api_key_auth, monkeypatch): + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail hook-b", seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + + async def reject(request: dict) -> None: + raise HTTPException(status_code=400, detail="model cannot route") + + with pytest.raises(HTTPException, match="model cannot route"): + await proxy_logging.pre_call_hook( + user_api_key_dict=make_user_api_key_auth(), + data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + call_type="acompletion", + before_guardrails=reject, + ) + + assert seen == ["hook-a"]