From 48f9fcbc951d4f6e173b5a4f575e6e1e28536a9e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:44:40 +0000 Subject: [PATCH 01/24] fix(proxy): validate requested model before running pre-call guardrails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 9 +- litellm/proxy/route_llm_request.py | 313 ++++++++++++------ .../proxy/test_common_request_processing.py | 99 ++++++ .../proxy/test_route_llm_request.py | 113 +++++++ 4 files changed, 438 insertions(+), 96 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2f39e6c71bc..6fd513a88cc 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -16,6 +16,7 @@ from typing import ( Protocol, TypeAlias, TypeVar, + cast, overload, runtime_checkable, ) @@ -93,7 +94,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( ) from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression -from litellm.proxy.route_llm_request import route_request +from litellm.proxy.route_llm_request import raise_if_model_not_routable, route_request from litellm.proxy.utils import ProxyLogging, _check_and_merge_model_level_guardrails from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict @@ -2103,6 +2104,12 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) + await raise_if_model_not_routable( + data=cast(dict[str, object], self.data), + llm_router=llm_router, + user_model=user_model, + route_type=route_type, + ) self.data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=self.data, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 20b4708c193..04182c917ee 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, cast import httpx from fastapi import HTTPException, status @@ -147,6 +147,125 @@ ROUTE_ENDPOINT_MAPPING: Final = { _AVAILABLE_MODELS_HINT: Final = "Call `/v1/models` to view available models for your key." +_EVAL_ROUTE_TYPES: Final = frozenset( + { + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + "acreate_run", + "alist_runs", + "aget_run", + "acancel_run", + "adelete_run", + } +) + +_CONTAINER_ROUTE_TYPES: Final = frozenset( + { + "acreate_container", + "alist_containers", + "aretrieve_container", + "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + } +) + +_INTERACTION_ROUTE_TYPES: Final = frozenset( + { + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", + } +) + +_MANAGED_AGENT_ROUTE_TYPES: Final = frozenset( + { + "acreate_agent", + "alist_agents", + "aget_agent", + "adelete_agent", + "alist_agent_versions", + } +) + +_MODEL_OPTIONAL_ROUTE_TYPES: Final = frozenset( + { + "avideo_list", + "avideo_status", + "avideo_content", + "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_content", + "avector_store_file_delete", + "acreate_skill", + "alist_skills", + "aget_skill", + "adelete_skill", + "aingest", + } +) + +_MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES: Final = frozenset( + { + "amoderation", + "aget_responses", + "adelete_responses", + "acancel_responses", + "alist_input_items", + "avector_store_create", + "avector_store_search", + "avector_store_retrieve", + "avector_store_list", + "avector_store_update", + "avector_store_delete", + "avector_store_file_create", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_content", + "avector_store_file_update", + "avector_store_file_delete", + "asearch", + "acreate_container", + "alist_containers", + "aretrieve_container", + "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + } +) + +_VIDEO_FALLBACK_ROUTE_TYPES: Final = frozenset( + { + "avideo_status", + "avideo_content", + "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", + } +) + +# Route types whose callers go through common_processing_pre_call_logic but +# never dispatch through route_request, so no model routing exists for them. +_NO_MODEL_ROUTING_ROUTE_TYPES: Final = frozenset({"apply_guardrail", "call_mcp_tool"}) + class ProxyModelNotFoundError(HTTPException): def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True): @@ -460,6 +579,95 @@ async def route_request( ) +def _router_can_route( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + route_type: str, +) -> bool: + """Pure predicate mirroring the branch structure of _route_request_single_attempt. + + Returns True when the existing routing tree would dispatch this request + somewhere instead of falling through to the final ProxyModelNotFoundError. + """ + if llm_router is None: + return True + if route_type in _NO_MODEL_ROUTING_ROUTE_TYPES: + return True + if "api_key" in data or "api_base" in data: + return True + model: Final = data.get("model") + if route_type == "acompletion" and isinstance(model, str) and "," in model: + return True + if "user_config" in data or "router_settings_override" in data: + return True + if ( + route_type in _EVAL_ROUTE_TYPES + or route_type in _CONTAINER_ROUTE_TYPES + or route_type in _INTERACTION_ROUTE_TYPES + or route_type in _MANAGED_AGENT_ROUTE_TYPES + ): + return True + if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and (model is None or model == ""): + return True + if not isinstance(model, str): + return True + + team_id: Final = get_team_id_from_data(dict(data)) + if team_id is not None and llm_router.map_team_model(model, team_id) is not None: + return True + + router_model_names: Final = cast(set[str], llm_router.model_names) + is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(dict(data)) + if ( + is_proxy_admin_without_team + and model not in router_model_names + and model in llm_router.team_public_model_names + ) or llm_router.is_recognized_model(model): + return True + + if model in router_model_names: + return False + if llm_router.router_general_settings.pass_through_all_models: + return True + if llm_router.default_deployment is not None or len(cast(dict[str, object], llm_router.pattern_router.patterns)) > 0: + return True + if model in cast(list[str], llm_router.deployment_names): + return True + if route_type in _MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES or route_type in _VIDEO_FALLBACK_ROUTE_TYPES: + return True + return _is_a2a_agent_model(model) + + +async def raise_if_model_not_routable( + data: dict[str, object], + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, +) -> None: + """Raise ProxyModelNotFoundError when route_request would reject the model. + + Lets pre-call steps (e.g. guardrails) skip work for requests that can never + route, and mirrors route_request's registry read-through retry. + """ + if _router_can_route(data=data, llm_router=llm_router, route_type=route_type): + return + requested_model: Final = data.get("model") + if isinstance(requested_model, str) and requested_model: + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + model_registry_read_through, + ) + + if await model_registry_read_through.attempt(requested_model) and _router_can_route( + data=data, llm_router=proxy_server.llm_router, route_type=route_type + ): + return + raise ProxyModelNotFoundError( + route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), + model_name=requested_model if isinstance(requested_model, str) else "", + ) + + async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request llm_router: LitellmRouter | None, @@ -540,19 +748,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) # Evals API: always route to litellm directly (not through router) # But extract model credentials if a model is provided - if route_type in [ - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - ]: + if route_type in _EVAL_ROUTE_TYPES: # If a model is provided, get its credentials from the router model: Final = data.get("model") if model and llm_router: @@ -578,54 +774,17 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr return getattr(litellm, f"{route_type}")(**data) # Skip model-based routing for container operations - if route_type in [ - "acreate_container", - "alist_containers", - "aretrieve_container", - "adelete_container", - "aupload_container_file", - "alist_container_files", - "aretrieve_container_file", - "adelete_container_file", - "aretrieve_container_file_content", - ]: + if route_type in _CONTAINER_ROUTE_TYPES: return getattr(llm_router, f"{route_type}")(**data) # Interactions API: create with agent, get/delete/cancel don't need model routing - if route_type in [ - "acreate_interaction", - "aget_interaction", - "adelete_interaction", - "acancel_interaction", - ]: + if route_type in _INTERACTION_ROUTE_TYPES: return getattr(llm_router, f"{route_type}")(**data) # Managed Agents API: these don't need model routing - if route_type in [ - "acreate_agent", - "alist_agents", - "aget_agent", - "adelete_agent", - "alist_agent_versions", - ]: + if route_type in _MANAGED_AGENT_ROUTE_TYPES: return getattr(llm_router, f"{route_type}")(**data) - if route_type in [ - "avideo_list", - "avideo_status", - "avideo_content", - "avideo_remix", - "avideo_create_character", - "avideo_get_character", - "avideo_edit", - "avideo_extension", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_content", - "avector_store_file_delete", - "acreate_skill", - "alist_skills", - "aget_skill", - "adelete_skill", - "aingest", - ] and (data.get("model") is None or data.get("model") == ""): + if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and ( + data.get("model") is None or data.get("model") == "" + ): # These endpoints don't need a model, use custom_llm_provider directly return getattr(litellm, f"{route_type}")(**data) @@ -651,46 +810,10 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr elif data["model"] in llm_router.deployment_names: # Only match deployment_names if no wildcard matched return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True) - elif route_type in [ - "amoderation", - "aget_responses", - "adelete_responses", - "acancel_responses", - "alist_input_items", - "avector_store_create", - "avector_store_search", - "avector_store_retrieve", - "avector_store_list", - "avector_store_update", - "avector_store_delete", - "avector_store_file_create", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_content", - "avector_store_file_update", - "avector_store_file_delete", - "asearch", - "acreate_container", - "alist_containers", - "aretrieve_container", - "adelete_container", - "aupload_container_file", - "alist_container_files", - "aretrieve_container_file", - "adelete_container_file", - "aretrieve_container_file_content", - ]: + elif route_type in _MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES: # These endpoints can work with or without model parameter return getattr(llm_router, f"{route_type}")(**data) - elif route_type in [ - "avideo_status", - "avideo_content", - "avideo_remix", - "avideo_create_character", - "avideo_get_character", - "avideo_edit", - "avideo_extension", - ]: + elif route_type in _VIDEO_FALLBACK_ROUTE_TYPES: # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), # try router first to allow for multi-deployment load balancing try: diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 4ac687625c2..5fa5aacfbe8 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -383,6 +383,105 @@ class TestProxyBaseLLMRequestProcessing: assert "litellm_logging_obj" not in persisted_body json.dumps(persisted_body) + @pytest.mark.asyncio + async def test_common_processing_pre_call_logic_rejects_unroutable_model_before_pre_call_hook(self, monkeypatch): + from types import SimpleNamespace + + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + processing_obj = ProxyBaseLLMRequestProcessing(data={}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return { + "model": "does-not-exist", + "messages": [{"role": "user", "content": "hi"}], + } + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + SimpleNamespace(attempt=AsyncMock(return_value=False)), + ) + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) + router = Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + } + ] + ) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=router, + ) + + assert exc_info.value.status_code == 400 + mock_proxy_logging_obj.pre_call_hook.assert_not_awaited() + + @pytest.mark.asyncio + async def test_common_processing_pre_call_logic_runs_pre_call_hook_for_routable_model(self, monkeypatch): + processing_obj = ProxyBaseLLMRequestProcessing(data={}) + mock_request = MagicMock(spec=Request) + mock_request.headers = {} + + async def mock_add_litellm_data_to_request(*args, **kwargs): + return { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + } + + async def mock_pre_call_hook(user_api_key_dict, data, call_type): + return data + + monkeypatch.setattr( + litellm.proxy.common_request_processing, + "add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ) + mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) + router = Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + } + ] + ) + + await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=mock_proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=router, + ) + + mock_proxy_logging_obj.pre_call_hook.assert_awaited_once() + @staticmethod def _guardrail_tag_budget_harness( monkeypatch, diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index f7021763a4d..73aa89f6294 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1308,3 +1308,116 @@ def test_proxy_model_not_found_error_keeps_the_raw_model_only_in_the_client_resp assert raw_model in error.detail["error"] assert raw_model not in error.spend_log_error_message assert error.spend_log_error_message.startswith("/chat/completions: Invalid model name passed in") + + +def _router_with_gpt4o_mini(): + import litellm + + return litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + } + ] + ) + + +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_raises_for_unknown_model(monkeypatch): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + SimpleNamespace(attempt=AsyncMock(return_value=False)), + ) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await raise_if_model_not_routable( + data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]}, + llm_router=_router_with_gpt4o_mini(), + user_model=None, + route_type="acompletion", + ) + + assert exc_info.value.status_code == 400 + assert "does-not-exist" in str(exc_info.value.detail) + + +@pytest.mark.parametrize( + "data, route_type, router_mutator", + [ + ( + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "openai/anything", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + "wildcard", + ), + ( + {"model": "does-not-exist", "api_key": "sk-user", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "does-not-exist,gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + None, + ), + ( + {"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]}, + "acompletion", + "pass_through_all_models", + ), + ( + {"model": "does-not-exist", "input": "hello"}, + "amoderation", + None, + ), + ], +) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_allows_routable_requests(monkeypatch, data, route_type, router_mutator): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + SimpleNamespace(attempt=AsyncMock(return_value=False)), + ) + + if router_mutator == "wildcard": + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-test"}, + } + ] + ) + else: + router = _router_with_gpt4o_mini() + if router_mutator == "pass_through_all_models": + # router_general_settings is shared across Router instances; monkeypatch restores it + monkeypatch.setattr(router.router_general_settings, "pass_through_all_models", True) + + await raise_if_model_not_routable( + data=data, + llm_router=router, + user_model=None, + route_type=route_type, + ) From 9b104e90705960b153af1780fa97e0379575940a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:46:27 +0000 Subject: [PATCH 02/24] refactor(proxy): drop explanatory comment on no-routing route types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/route_llm_request.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 04182c917ee..2c981d4c316 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -262,8 +262,6 @@ _VIDEO_FALLBACK_ROUTE_TYPES: Final = frozenset( } ) -# Route types whose callers go through common_processing_pre_call_logic but -# never dispatch through route_request, so no model routing exists for them. _NO_MODEL_ROUTING_ROUTE_TYPES: Final = frozenset({"apply_guardrail", "call_mcp_tool"}) From 1b701901bc5f1c637542caa05435a950aaf666bc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 17:50:21 +0000 Subject: [PATCH 03/24] style(proxy): ruff format route_llm_request Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/route_llm_request.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 2c981d4c316..d827bb7c6b1 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -617,9 +617,7 @@ def _router_can_route( router_model_names: Final = cast(set[str], llm_router.model_names) is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(dict(data)) if ( - is_proxy_admin_without_team - and model not in router_model_names - and model in llm_router.team_public_model_names + is_proxy_admin_without_team and model not in router_model_names and model in llm_router.team_public_model_names ) or llm_router.is_recognized_model(model): return True @@ -627,7 +625,10 @@ def _router_can_route( return False if llm_router.router_general_settings.pass_through_all_models: return True - if llm_router.default_deployment is not None or len(cast(dict[str, object], llm_router.pattern_router.patterns)) > 0: + if ( + llm_router.default_deployment is not None + or len(cast(dict[str, object], llm_router.pattern_router.patterns)) > 0 + ): return True if model in cast(list[str], llm_router.deployment_names): return True @@ -780,9 +781,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr # Managed Agents API: these don't need model routing if route_type in _MANAGED_AGENT_ROUTE_TYPES: return getattr(llm_router, f"{route_type}")(**data) - if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and ( - data.get("model") is None or data.get("model") == "" - ): + if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and (data.get("model") is None or data.get("model") == ""): # These endpoints don't need a model, use custom_llm_provider directly return getattr(litellm, f"{route_type}")(**data) From 6882a2e88e702fd8d708a3d18e2f506e68ab6eba Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:06:28 +0000 Subject: [PATCH 04/24] fix(proxy): satisfy strict lint gates in model routability check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/common_request_processing.py | 3 +-- litellm/proxy/route_llm_request.py | 27 ++++++++++++++-------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6fd513a88cc..55084835159 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -16,7 +16,6 @@ from typing import ( Protocol, TypeAlias, TypeVar, - cast, overload, runtime_checkable, ) @@ -2105,7 +2104,7 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) await raise_if_model_not_routable( - data=cast(dict[str, object], self.data), + data=self.data, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # loosely typed llm_router=llm_router, user_model=user_model, route_type=route_type, diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index d827bb7c6b1..68365271864 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -1,6 +1,6 @@ import asyncio from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, Final, Literal, cast +from typing import TYPE_CHECKING, Any, Final, Literal import httpx from fastapi import HTTPException, status @@ -610,27 +610,36 @@ def _router_can_route( if not isinstance(model, str): return True + return _router_model_group_routable(model=model, llm_router=llm_router, data=data, route_type=route_type) + + +def _router_model_group_routable( + model: str, + llm_router: LitellmRouter, + data: Mapping[str, object], + route_type: str, +) -> bool: team_id: Final = get_team_id_from_data(dict(data)) if team_id is not None and llm_router.map_team_model(model, team_id) is not None: return True - router_model_names: Final = cast(set[str], llm_router.model_names) is_proxy_admin_without_team: Final = team_id is None and _is_proxy_admin_request(dict(data)) if ( - is_proxy_admin_without_team and model not in router_model_names and model in llm_router.team_public_model_names + is_proxy_admin_without_team + and model not in llm_router.model_names # pyright: ignore[reportUnknownMemberType] # bare set + and model in llm_router.team_public_model_names ) or llm_router.is_recognized_model(model): return True - if model in router_model_names: + if model in llm_router.model_names: # pyright: ignore[reportUnknownMemberType] # bare set return False if llm_router.router_general_settings.pass_through_all_models: return True - if ( - llm_router.default_deployment is not None - or len(cast(dict[str, object], llm_router.pattern_router.patterns)) > 0 - ): + if llm_router.default_deployment is not None: return True - if model in cast(list[str], llm_router.deployment_names): + if llm_router.pattern_router.patterns: # pyright: ignore[reportUnknownMemberType] # untyped dict + return True + if model in llm_router.deployment_names: # pyright: ignore[reportUnknownMemberType] # bare list return True if route_type in _MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES or route_type in _VIDEO_FALLBACK_ROUTE_TYPES: return True From e72d2f3b77e1bccf0787add2c4c885387fdbe292 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:12:24 +0000 Subject: [PATCH 05/24] test(proxy): assert routable predicate returns none for allowed requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/test_litellm/proxy/test_route_llm_request.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_litellm/proxy/test_route_llm_request.py b/tests/test_litellm/proxy/test_route_llm_request.py index 73aa89f6294..b67f3a980d6 100644 --- a/tests/test_litellm/proxy/test_route_llm_request.py +++ b/tests/test_litellm/proxy/test_route_llm_request.py @@ -1415,9 +1415,12 @@ async def test_raise_if_model_not_routable_allows_routable_requests(monkeypatch, # router_general_settings is shared across Router instances; monkeypatch restores it monkeypatch.setattr(router.router_general_settings, "pass_through_all_models", True) - await raise_if_model_not_routable( - data=data, - llm_router=router, - user_model=None, - route_type=route_type, + assert ( + await raise_if_model_not_routable( + data=data, + llm_router=router, + user_model=None, + route_type=route_type, + ) + is None ) From 52992e42aeabe35432d354294809de5d0fcffd6c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:34:16 +0000 Subject: [PATCH 06/24] fix(proxy): mirror routerless and fully-blocked paths in routability check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/route_llm_request.py | 12 +++++++----- .../proxy/test_common_request_processing.py | 9 ++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 68365271864..3605aa9c1c7 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -580,6 +580,7 @@ async def route_request( def _router_can_route( data: Mapping[str, object], llm_router: LitellmRouter | None, + user_model: str | None, route_type: str, ) -> bool: """Pure predicate mirroring the branch structure of _route_request_single_attempt. @@ -587,17 +588,18 @@ def _router_can_route( Returns True when the existing routing tree would dispatch this request somewhere instead of falling through to the final ProxyModelNotFoundError. """ - if llm_router is None: - return True if route_type in _NO_MODEL_ROUTING_ROUTE_TYPES: return True if "api_key" in data or "api_base" in data: return True model: Final = data.get("model") - if route_type == "acompletion" and isinstance(model, str) and "," in model: + if route_type == "acompletion" and isinstance(model, str) and "," in model and llm_router is not None: return True if "user_config" in data or "router_settings_override" in data: return True + if llm_router is None: + return user_model is not None or route_type == "allm_passthrough_route" + _raise_if_model_fully_blocked(llm_router=llm_router, model_name=model, team_id=get_team_id_from_data(dict(data))) if ( route_type in _EVAL_ROUTE_TYPES or route_type in _CONTAINER_ROUTE_TYPES @@ -657,7 +659,7 @@ async def raise_if_model_not_routable( Lets pre-call steps (e.g. guardrails) skip work for requests that can never route, and mirrors route_request's registry read-through retry. """ - if _router_can_route(data=data, llm_router=llm_router, route_type=route_type): + if _router_can_route(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): return requested_model: Final = data.get("model") if isinstance(requested_model, str) and requested_model: @@ -667,7 +669,7 @@ async def raise_if_model_not_routable( ) if await model_registry_read_through.attempt(requested_model) and _router_can_route( - data=data, llm_router=proxy_server.llm_router, route_type=route_type + data=data, llm_router=proxy_server.llm_router, user_model=user_model, route_type=route_type ): return raise ProxyModelNotFoundError( diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py index 5fa5aacfbe8..8f2d0e55c50 100644 --- a/tests/test_litellm/proxy/test_common_request_processing.py +++ b/tests/test_litellm/proxy/test_common_request_processing.py @@ -300,6 +300,7 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) route_type = "acompletion" # Call the actual method. @@ -313,6 +314,7 @@ class TestProxyBaseLLMRequestProcessing: proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, route_type=route_type, + llm_router=self._router_with_free_and_paid_models(), ) mock_proxy_logging_obj.pre_call_hook.assert_called_once() @@ -364,14 +366,17 @@ class TestProxyBaseLLMRequestProcessing: "add_litellm_data_to_request", mock_add_litellm_data_to_request, ) + mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) returned_data, _ = await processing_obj.common_processing_pre_call_logic( request=mock_request, general_settings={}, user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), proxy_logging_obj=mock_proxy_logging_obj, - proxy_config=MagicMock(spec=ProxyConfig), + proxy_config=mock_proxy_config, route_type="acompletion", + llm_router=self._router_with_free_and_paid_models(), ) persisted_body = returned_data["proxy_server_request"]["body"] @@ -1905,6 +1910,7 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) route_type = "acompletion" ( @@ -1917,6 +1923,7 @@ class TestProxyBaseLLMRequestProcessing: proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, route_type=route_type, + llm_router=self._router_with_free_and_paid_models(), ) # Verify queue_time_seconds is set and non-negative. Ends at start_time From 5a2031ec5edac2f340f0204a9afa8fec36a7de7d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:45:33 +0000 Subject: [PATCH 07/24] fix(proxy): skip routability gate for endpoints that resolve their own target Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/route_llm_request.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 3605aa9c1c7..5a1bd4874ff 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -262,7 +262,27 @@ _VIDEO_FALLBACK_ROUTE_TYPES: Final = frozenset( } ) -_NO_MODEL_ROUTING_ROUTE_TYPES: Final = frozenset({"apply_guardrail", "call_mcp_tool"}) +_NO_MODEL_ROUTING_ROUTE_TYPES: Final = frozenset( + { + "apply_guardrail", + "call_mcp_tool", + # Files, fine-tuning, batch, and A2A endpoints resolve their own target + # (file id, job id, or agent) after pre-call processing and never reach + # _route_request_single_attempt. + "afile_content", + "afile_retrieve", + "afile_delete", + "acreate_fine_tuning_job", + "aretrieve_fine_tuning_job", + "alist_fine_tuning_jobs", + "acancel_fine_tuning_job", + "acreate_batch", + "aretrieve_batch", + "alist_batches", + "acancel_batch", + "asend_message", + } +) class ProxyModelNotFoundError(HTTPException): From 4a287aa468b4ba716131bb3869cc20e41d22999e Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:58:05 -0700 Subject: [PATCH 08/24] Run non-guardrail pre-call hooks before the model check and redact requests rejected before guardrails Hooks that rewrite the model (prompt templates, custom pre-call hooks) run before the routability check; guardrails and content enforcers run after it on the rewritten request. Websocket callers without a router skip the check. The check reports a missing required body param or disallowed mock params before the model, as route_request does, and retries the registry read-through only for a model those hooks changed. A request rejected before the pre-call guardrails that apply to it ran has its content, its proxy_server_request snapshot and its message logging redacted, so failure callbacks, post-call failure hooks and spend rows never keep content a masking guardrail would have removed. --- litellm/proxy/common_request_processing.py | 91 ++++++- litellm/proxy/route_llm_request.py | 72 ++++-- litellm/proxy/utils.py | 9 + .../proxy/test_common_request_processing.py | 243 +++++++++++++++++- tests/unit/proxy/test_route_llm_request.py | 49 +++- .../utils/proxy_logging/test_pre_call_hook.py | 30 +++ 6 files changed, 468 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index bf882cf2727..e72db7f721e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -43,6 +43,7 @@ from litellm.constants import ( MAX_LITELLM_CALL_ID_LENGTH, MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, NON_INFERENCE_CALL_TYPES, + REDACTED_BY_LITELLM, RETURN_RAW_MODEL_NAME_METADATA_KEY, STREAM_SSE_DATA_PREFIX, STREAM_SSE_KEEPALIVE_PING_BYTES, @@ -118,6 +119,7 @@ from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( + is_model_routable, raise_if_model_not_routable, route_request, ) @@ -254,6 +256,7 @@ from litellm.proxy.litellm_pre_call_utils import ( ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.utils import ( + CallTypesLiteral, ModelResponse, ModelResponseStream, StandardLoggingPayloadErrorInformation, @@ -1736,7 +1739,7 @@ def _timing_values( class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict): - self.data = data + self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place self._tags_before_guardrails: frozenset[str] | None = None @property @@ -2252,7 +2255,7 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) - await raise_if_model_not_routable( + model_routable: Final = llm_router is None or await is_model_routable( data=self.data, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # loosely typed llm_router=llm_router, user_model=user_model, @@ -2261,11 +2264,14 @@ class ProxyBaseLLMRequestProcessing: prefetch_model = self.data.get("model") if llm_router is not None and isinstance(prefetch_model, str): llm_router.arm_routing_read_prefetch(prefetch_model, self.data) - self.data = await proxy_logging_obj.pre_call_hook( + await self._run_pre_call_hooks( + proxy_logging_obj=proxy_logging_obj, user_api_key_dict=user_api_key_dict, - data=self.data, call_type=route_type, skip_guardrails=skip_guardrails, + model_routable=model_routable, + llm_router=llm_router, + user_model=user_model, ) await _enforce_guardrail_added_tag_budgets( data=self.data, @@ -2293,6 +2299,81 @@ class ProxyBaseLLMRequestProcessing: return self.data, logging_obj + async def _run_pre_call_hooks( + self, + proxy_logging_obj: ProxyLogging, + user_api_key_dict: UserAPIKeyAuth, + call_type: CallTypesLiteral, + skip_guardrails: bool, + model_routable: bool, + llm_router: Router | None, + user_model: str | None, + ) -> None: + """Run the pre-call hooks on self.data, with guardrails last when the requested model cannot route yet. + + For such a model, hooks other than guardrails and content enforcers run first and may + rewrite the model. A model that still cannot route raises the routing error before any + guardrail runs; otherwise the guardrails run on the rewritten request. The registry + read-through is retried only for a model those hooks changed. + """ + if model_routable: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + skip_guardrails=skip_guardrails, + ) + return + model_before_hooks: Final = self.data.get("model") + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + skip_guardrails=True, + skip_content_enforcers=True, + ) + try: + await raise_if_model_not_routable( + data=self.data, + llm_router=llm_router, + user_model=user_model, + route_type=call_type, + read_through=self.data.get("model") != model_before_hooks, + ) + except Exception: + if not skip_guardrails: + self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) + raise + if not skip_guardrails: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + guardrails_only=True, + ) + + def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None: + """Redact the content of a request rejected before the pre-call guardrails that apply to it ran. + + The request body, its proxy_server_request snapshot and message logging all carry the + redaction marker, so failure callbacks, post-call failure hooks and spend rows never see + content a masking guardrail would have rewritten. + """ + request_metadata: Final = dict[str, object]() + for bucket in _request_metadata_buckets(self.data): + request_metadata.update(bucket) + if not proxy_logging_obj.has_pre_call_guardrails(request_metadata): + return + if "messages" in self.data: + self.data["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] + for content_key in ("prompt", "input"): + if content_key in self.data: + self.data[content_key] = "" + refresh_proxy_server_request_body_snapshot(self.data) + logging_obj: Final = self.data.get("litellm_logging_obj") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.standard_callback_dynamic_params["turn_off_message_logging"] = True + async def _pre_call_with_fallbacks( self, request: Request, @@ -2520,7 +2601,7 @@ class ProxyBaseLLMRequestProcessing: "Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s", len(_payload_str), MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, - (list(self.data.keys()) if isinstance(self.data, dict) else type(self.data).__name__), + list(self.data.keys()), ) else: verbose_proxy_logger.debug( diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 4d11c66b015..819470eaedf 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -820,30 +820,68 @@ def _router_model_group_routable( return _is_a2a_agent_model(model) -async def raise_if_model_not_routable( - data: dict[str, object], +def _router_can_route_unless_blocked( + data: Mapping[str, object], llm_router: LitellmRouter | None, user_model: str | None, route_type: str, -) -> None: - """Raise ProxyModelNotFoundError when route_request would reject the model. +) -> bool: + try: + return _router_can_route(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type) + except litellm.PermissionDeniedError: + return False - Lets pre-call steps (e.g. guardrails) skip work for requests that can never - route, and mirrors route_request's registry read-through retry. + +async def is_model_routable( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, + read_through: bool = True, +) -> bool: + """Whether route_request would dispatch this request, including its registry read-through retry. + + A fully blocked model counts as not routable. read_through=False skips the registry retry. """ - if _router_can_route(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): - return + if _router_can_route_unless_blocked(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): + return True requested_model: Final = data.get("model") - if isinstance(requested_model, str) and requested_model: - from litellm.proxy import proxy_server - from litellm.proxy.common_utils.registry_read_through import ( - model_registry_read_through, - ) + if not read_through or not isinstance(requested_model, str) or not requested_model: + return False + from litellm.proxy import proxy_server + from litellm.proxy.common_utils.registry_read_through import ( + model_registry_read_through, + ) - if await model_registry_read_through.attempt(requested_model) and _router_can_route( - data=data, llm_router=proxy_server.llm_router, user_model=user_model, route_type=route_type - ): - return + return await model_registry_read_through.attempt(requested_model) and _router_can_route_unless_blocked( + data=data, llm_router=proxy_server.llm_router, user_model=user_model, route_type=route_type + ) + + +async def raise_if_model_not_routable( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, + read_through: bool = True, +) -> None: + """Raise the error route_request would raise for a model it cannot dispatch. + + The body checks route_request runs first come first: a missing required param, then + disallowed mock testing params. After them, PermissionDeniedError (403) for a fully blocked + model and ProxyModelNotFoundError (400) otherwise. read_through=False skips the registry retry. + """ + if await is_model_routable( + data=data, llm_router=llm_router, user_model=user_model, route_type=route_type, read_through=read_through + ): + return + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router) + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) + requested_model: Final = data.get("model") + if llm_router is not None: + _raise_if_model_fully_blocked( + llm_router=llm_router, model_name=requested_model, team_id=get_team_id_from_data(dict(data)) + ) raise ProxyModelNotFoundError( route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), model_name=requested_model if isinstance(requested_model, str) else "", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 302f55eadf4..1ca9f4b08a6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2426,6 +2426,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + skip_content_enforcers: bool = False, ) -> None: pass @@ -2438,6 +2439,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + skip_content_enforcers: bool = False, ) -> dict: pass @@ -2449,6 +2451,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, + skip_content_enforcers: bool = False, ) -> dict | None: """ Allows users to modify/reject the incoming request to the proxy, without having to deal with parsing Request body. @@ -2461,11 +2464,16 @@ class ProxyLogging: With ``guardrails_only`` the walk is limited to guardrails and guardrail pipelines: rate limiting, budget accounting, prompt templates and hanging-request alerting are skipped. Use it to scan a payload that is not itself a request, such as one record of a batch file. + + ``skip_content_enforcers`` also skips the non-guardrail callbacks that set + ``enforces_request_content``, the ones ``guardrails_only`` keeps. """ verbose_proxy_logger.debug("Inside Proxy Logging Pre-call hook!") if guardrails_only and skip_guardrails: raise ValueError("guardrails_only and skip_guardrails are mutually exclusive") + if guardrails_only and skip_content_enforcers: + raise ValueError("guardrails_only and skip_content_enforcers are mutually exclusive") if not guardrails_only: self._init_response_taking_too_long_task(data=data) @@ -2581,6 +2589,7 @@ class ProxyLogging: _callback is not None and isinstance(_callback, CustomLogger) and (not guardrails_only or _callback.enforces_request_content) + and not (skip_content_enforcers and _callback.enforces_request_content) and "async_pre_call_hook" in vars(_callback.__class__) and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook ): diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index eb8f7dd2a93..11a5481ff2a 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -29,6 +29,7 @@ from litellm.constants import ( RETURN_RAW_MODEL_NAME_METADATA_KEY, STREAM_SSE_KEEPALIVE_PING_BYTES, ) +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.opentelemetry import UserAPIKeyAuth from litellm.proxy.common_request_processing import ( @@ -67,6 +68,7 @@ from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.utils import ProxyLogging from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment +from litellm.types.guardrails import GuardrailEventHooks def test_attach_guardrail_information_copies_recorded_entries_onto_model_response(): @@ -621,7 +623,7 @@ class TestProxyBaseLLMRequestProcessing: SimpleNamespace(attempt=AsyncMock(return_value=False)), ) mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) - mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data) + mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) mock_proxy_config = MagicMock(spec=ProxyConfig) mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) router = Router( @@ -645,7 +647,10 @@ class TestProxyBaseLLMRequestProcessing: ) assert exc_info.value.status_code == 400 - mock_proxy_logging_obj.pre_call_hook.assert_not_awaited() + mock_proxy_logging_obj.pre_call_hook.assert_awaited_once() + hook_kwargs: Final = mock_proxy_logging_obj.pre_call_hook.await_args.kwargs + assert hook_kwargs["skip_guardrails"] is True + assert hook_kwargs["skip_content_enforcers"] is True @pytest.mark.asyncio async def test_common_processing_pre_call_logic_runs_pre_call_hook_for_routable_model(self, monkeypatch): @@ -659,7 +664,7 @@ class TestProxyBaseLLMRequestProcessing: "messages": [{"role": "user", "content": "hi"}], } - async def mock_pre_call_hook(user_api_key_dict, data, call_type): + async def mock_pre_call_hook(user_api_key_dict, data, call_type, skip_guardrails=False): return data monkeypatch.setattr( @@ -10749,3 +10754,235 @@ class TestStreamingContainerOwnershipRecordedBeforeDone: assert tuple(chunk for chunk, _ in observed) == self.CHUNKS assert tuple(count for _, count in observed) == (0, 0, 0, 0) recorder.assert_awaited_once() + + +class _CountingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="counting-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True) + self.models_seen: list[object] = [] + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.models_seen.append(data.get("model")) + return data + + +class _CountingContentEnforcer(CustomLogger): + enforces_request_content = True + + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + return data + + +class _ModelRewritingHook(CustomLogger): + def __init__(self, source_model: str = "virtual-model", target_model: str = "gpt-4o-mini") -> None: + super().__init__() + self.source_model = source_model + self.target_model = target_model + self.calls = 0 + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + if data.get("model") == self.source_model: + data["model"] = self.target_model + return data + + +class TestPreCallHooksForUnroutableModel: + @staticmethod + async def _run( + monkeypatch, + model: str, + llm_router: Router | None, + processor: ProxyBaseLLMRequestProcessing | None = None, + skip_guardrails: bool = False, + read_through_attempt: AsyncMock | None = None, + ) -> dict: + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + async def mock_add_litellm_data_to_request(*args, **kwargs): + messages: Final = [{"role": "user", "content": "email jane@example.com"}] + return { + "model": model, + "messages": messages, + "proxy_server_request": {"body": {"model": model, "messages": messages}}, + } + + monkeypatch.setattr( + litellm.proxy.common_request_processing, "add_litellm_data_to_request", mock_add_litellm_data_to_request + ) + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + SimpleNamespace(attempt=read_through_attempt or AsyncMock(return_value=False)), + ) + mock_request: Final = MagicMock(spec=Request) + mock_request.headers = {} + mock_proxy_config: Final = MagicMock(spec=ProxyConfig) + mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) + proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) + data, _ = await (processor or ProxyBaseLLMRequestProcessing(data={})).common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=ProxyUserAPIKeyAuth(), + proxy_logging_obj=proxy_logging_obj, + proxy_config=mock_proxy_config, + route_type="acompletion", + llm_router=llm_router, + skip_guardrails=skip_guardrails, + ) + return data + + @staticmethod + def _router() -> Router: + return Router( + model_list=[ + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + { + "model_name": "blocked-model", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, + "model_info": {"blocked": True}, + }, + ] + ) + + @pytest.mark.asyncio + async def test_unknown_model_runs_other_hooks_but_no_guardrail_or_content_enforcer(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router()) + + assert exc_info.value.status_code == 400 + assert hook.calls == 1 + assert guardrail.models_seen == [] + assert enforcer.calls == 0 + + @pytest.mark.asyncio + async def test_hook_that_rewrites_unknown_model_to_routable_one_still_gets_guardrails(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert hook.calls == 1 + assert guardrail.models_seen == ["gpt-4o-mini"] + assert enforcer.calls == 1 + + @pytest.mark.asyncio + async def test_routable_model_runs_every_hook_once(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + enforcer: Final = _CountingContentEnforcer() + hook: Final = _ModelRewritingHook() + monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer]) + + await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router()) + + assert (hook.calls, guardrail.models_seen, enforcer.calls) == (1, ["gpt-4o-mini"], 1) + + @pytest.mark.asyncio + async def test_fully_blocked_model_is_a_403_without_guardrails(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + with pytest.raises(litellm.PermissionDeniedError): + await self._run(monkeypatch, model="blocked-model", llm_router=self._router()) + + assert guardrail.models_seen == [] + + @pytest.mark.asyncio + async def test_hook_that_rewrites_blocked_model_to_routable_one_is_served(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(source_model="blocked-model"), guardrail]) + + data: Final = await self._run(monkeypatch, model="blocked-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert guardrail.models_seen == ["gpt-4o-mini"] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("callbacks", "skip_guardrails", "expected"), + [ + pytest.param(lambda: [_CountingGuardrail()], False, True, id="guardrail-applies"), + pytest.param(lambda: [_CountingContentEnforcer()], False, True, id="content-enforcer"), + pytest.param(lambda: [_ModelRewritingHook()], False, None, id="no-guardrail"), + pytest.param(lambda: [_CountingGuardrail()], True, None, id="guardrails-skipped-by-caller"), + ], + ) + async def test_rejected_request_is_redacted_only_when_guardrails_were_bypassed( + self, monkeypatch, callbacks, skip_guardrails, expected + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", callbacks()) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + skip_guardrails=skip_guardrails, + ) + + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is expected + assert logging_obj.model_call_details["standard_callback_dynamic_params"] is ( + logging_obj.standard_callback_dynamic_params + ) + expected_content: Final = "redacted-by-litellm" if expected else "email jane@example.com" + assert processor.data["messages"] == [{"role": "user", "content": expected_content}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + + @pytest.mark.asyncio + async def test_unknown_model_hooks_left_unchanged_gets_one_registry_read_through(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()]) + attempt: Final = AsyncMock(return_value=False) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), read_through_attempt=attempt) + + assert [call.args[0] for call in attempt.await_args_list] == ["does-not-exist"] + + @pytest.mark.asyncio + async def test_model_rewritten_by_a_hook_is_read_through_under_its_new_name(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr( + litellm, + "callbacks", + [_ModelRewritingHook(source_model="unknown-virtual", target_model="db-only-model")], + ) + attempt: Final = AsyncMock(return_value=False) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), read_through_attempt=attempt) + + assert [call.args[0] for call in attempt.await_args_list] == ["unknown-virtual", "db-only-model"] + + @pytest.mark.asyncio + async def test_no_router_skips_the_routability_check(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + + data: Final = await self._run(monkeypatch, model="router-alias-only-known-at-dispatch", llm_router=None) + + assert data["model"] == "router-alias-only-known-at-dispatch" + assert guardrail.models_seen == ["router-alias-only-known-at-dispatch"] diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 1aa50bfb82a..eacceaa5727 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -7,7 +7,12 @@ from unittest.mock import MagicMock from fastapi import HTTPException -from litellm.proxy.route_llm_request import ProxyModelNotFoundError, route_request +from litellm.proxy.route_llm_request import ( + MockTestingParamsDisabledError, + ProxyMissingRequiredParamError, + ProxyModelNotFoundError, + route_request, +) @pytest.mark.parametrize( @@ -1747,6 +1752,48 @@ async def test_raise_if_model_not_routable_raises_for_unknown_model(monkeypatch) assert "does-not-exist" in str(exc_info.value.detail) +@pytest.mark.parametrize( + ("data", "route_type", "expected_error"), + [ + pytest.param( + {"model": "does-not-exist"}, + "atext_completion", + ProxyMissingRequiredParamError, + id="missing-prompt", + ), + pytest.param( + {"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], "mock_timeout": True}, + "acompletion", + MockTestingParamsDisabledError, + id="mock-params-disallowed", + ), + ], +) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_reports_body_errors_before_the_model( + monkeypatch, data, route_type, expected_error +): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + SimpleNamespace(attempt=AsyncMock(return_value=False)), + ) + + with pytest.raises(expected_error): + await raise_if_model_not_routable( + data=data, + llm_router=_router_with_gpt4o_mini(), + user_model=None, + route_type=route_type, + ) + + @pytest.mark.parametrize( "data, route_type, router_mutator", [ diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index b7fc8c6dbeb..7174ebdbc47 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -995,3 +995,33 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +@pytest.mark.asyncio +async def test_skip_content_enforcers_skips_enforcers_but_not_other_hooks(proxy_logging, monkeypatch): + enforcer = _Enforcer() + accountant = _Accountant() + monkeypatch.setattr(litellm, "callbacks", [enforcer, accountant]) + + await proxy_logging.pre_call_hook( + user_api_key_dict=MagicMock(), + data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + call_type="acompletion", + skip_guardrails=True, + skip_content_enforcers=True, + ) + + assert enforcer.calls == 0 + assert accountant.calls == 1 + + +@pytest.mark.asyncio +async def test_skip_content_enforcers_and_guardrails_only_are_mutually_exclusive(proxy_logging): + with pytest.raises(ValueError, match="mutually exclusive"): + await proxy_logging.pre_call_hook( + user_api_key_dict=MagicMock(), + data={"model": "m"}, + call_type="acompletion", + guardrails_only=True, + skip_content_enforcers=True, + ) From 1e8c4d7c815701ed4370e0ebfdbe85a90244e374 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:20:30 -0700 Subject: [PATCH 09/24] JSON-encode request keys in the oversized-payload debug log --- litellm/proxy/common_request_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e72db7f721e..18c32252dd4 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2601,7 +2601,7 @@ class ProxyBaseLLMRequestProcessing: "Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s", len(_payload_str), MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, - list(self.data.keys()), + json.dumps(list(self.data.keys())), ) else: verbose_proxy_logger.debug( From 01445b3a35049bf57b82c342d17802c383c4fe6c Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 04:35:17 -0700 Subject: [PATCH 10/24] Strip line breaks from request keys in the oversized-payload debug log --- litellm/proxy/common_request_processing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 18c32252dd4..d5c5ea87724 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -253,6 +253,7 @@ from litellm.proxy.litellm_pre_call_utils import ( add_litellm_data_to_request, refresh_proxy_server_request_body_snapshot, reject_url_valued_destination, + sanitize_for_log, ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval from litellm.types.utils import ( @@ -2601,7 +2602,7 @@ class ProxyBaseLLMRequestProcessing: "Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s", len(_payload_str), MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG, - json.dumps(list(self.data.keys())), + sanitize_for_log(list(self.data.keys())), ) else: verbose_proxy_logger.debug( From 0096a74dce46cc375a049cdd4bb4e611d66532e3 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 11:19:55 -0700 Subject: [PATCH 11/24] Redact every guardrail-scanned field of a request rejected before guardrails --- litellm/proxy/common_request_processing.py | 25 +++++++- .../proxy/test_common_request_processing.py | 63 +++++++++++++++++-- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index d5c5ea87724..a3581e44566 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2356,9 +2356,12 @@ class ProxyBaseLLMRequestProcessing: def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None: """Redact the content of a request rejected before the pre-call guardrails that apply to it ran. - The request body, its proxy_server_request snapshot and message logging all carry the - redaction marker, so failure callbacks, post-call failure hooks and spend rows never see - content a masking guardrail would have rewritten. + Covers every request field a pre-call guardrail can rewrite on the routes that reach this + check: chat and Anthropic messages and system, completion prompt, Responses input and + instructions, embedding and speech input, and Gemini contents and system instruction. The + request body, its proxy_server_request snapshot and message logging all carry the redaction + marker, so failure callbacks, post-call failure hooks and spend rows never see content a + masking guardrail would have rewritten. """ request_metadata: Final = dict[str, object]() for bucket in _request_metadata_buckets(self.data): @@ -2367,9 +2370,25 @@ class ProxyBaseLLMRequestProcessing: return if "messages" in self.data: self.data["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] + if "contents" in self.data: + self.data["contents"] = [{"role": "user", "parts": [{"text": REDACTED_BY_LITELLM}]}] for content_key in ("prompt", "input"): if content_key in self.data: self.data[content_key] = "" + for content_key in ("system", "instructions"): + if content_key in self.data: + self.data[content_key] = REDACTED_BY_LITELLM + redacted_system_instruction: Final = {"parts": [{"text": REDACTED_BY_LITELLM}]} + system_instruction_keys: Final = ("systemInstruction", "system_instruction") + for content_key in system_instruction_keys: + if content_key in self.data: + self.data[content_key] = redacted_system_instruction + config: Final = self.data.get("config") + if isinstance(config, Mapping): + self.data["config"] = { + **config, + **{key: redacted_system_instruction for key in system_instruction_keys if key in config}, + } refresh_proxy_server_request_body_snapshot(self.data) logging_obj: Final = self.data.get("litellm_logging_obj") if isinstance(logging_obj, LiteLLMLoggingObj): diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 11a5481ff2a..f6a0bef00fb 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10801,17 +10801,15 @@ class TestPreCallHooksForUnroutableModel: processor: ProxyBaseLLMRequestProcessing | None = None, skip_guardrails: bool = False, read_through_attempt: AsyncMock | None = None, + extra_body: Mapping[str, object] | None = None, ) -> dict: from litellm.proxy.common_utils import registry_read_through from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache async def mock_add_litellm_data_to_request(*args, **kwargs): messages: Final = [{"role": "user", "content": "email jane@example.com"}] - return { - "model": model, - "messages": messages, - "proxy_server_request": {"body": {"model": model, "messages": messages}}, - } + body: Final = {"model": model, "messages": messages, **copy.deepcopy(dict(extra_body or {}))} + return {**body, "proxy_server_request": {"body": copy.deepcopy(body)}} monkeypatch.setattr( litellm.proxy.common_request_processing, "add_litellm_data_to_request", mock_add_litellm_data_to_request @@ -10949,6 +10947,61 @@ class TestPreCallHooksForUnroutableModel: assert processor.data["messages"] == [{"role": "user", "content": expected_content}] assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("extra_body", "expected_body"), + [ + pytest.param( + {"system": [{"type": "text", "text": "email jane@example.com"}]}, + {"system": "redacted-by-litellm"}, + id="anthropic-system", + ), + pytest.param( + {"instructions": "email jane@example.com"}, + {"instructions": "redacted-by-litellm"}, + id="responses-instructions", + ), + pytest.param( + {"contents": [{"role": "user", "parts": [{"text": "email jane@example.com"}]}]}, + {"contents": [{"role": "user", "parts": [{"text": "redacted-by-litellm"}]}]}, + id="gemini-contents", + ), + pytest.param( + {"systemInstruction": {"parts": [{"text": "email jane@example.com"}]}}, + {"systemInstruction": {"parts": [{"text": "redacted-by-litellm"}]}}, + id="gemini-system-instruction", + ), + pytest.param( + {"config": {"system_instruction": {"parts": [{"text": "email jane@example.com"}]}, "temperature": 0}}, + {"config": {"system_instruction": {"parts": [{"text": "redacted-by-litellm"}]}, "temperature": 0}}, + id="gemini-config-system-instruction", + ), + pytest.param( + {"prompt": "email jane@example.com", "input": ["email jane@example.com"], "max_tokens": 5}, + {"prompt": "", "input": "", "max_tokens": 5}, + id="prompt-and-input", + ), + ], + ) + async def test_rejected_request_redacts_every_guardrail_scanned_field(self, monkeypatch, extra_body, expected_body): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + extra_body=extra_body, + ) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert {key: processor.data[key] for key in expected_body} == expected_body + assert {key: snapshot[key] for key in expected_body} == expected_body + @pytest.mark.asyncio async def test_unknown_model_hooks_left_unchanged_gets_one_registry_read_through(self, monkeypatch): from litellm.proxy.route_llm_request import ProxyModelNotFoundError From 22d7adf908b774d6abc219978be3a03e867e5bc3 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:09:52 -0700 Subject: [PATCH 12/24] Redact a request a pre-call hook rejects before guardrails, and drive the routability tests through real request preparation --- litellm/proxy/common_request_processing.py | 19 +- .../proxy/test_common_request_processing.py | 192 ++++++------------ tests/unit/proxy/test_route_llm_request.py | 36 +--- 3 files changed, 78 insertions(+), 169 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a3581e44566..d285c8b3630 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2314,8 +2314,9 @@ class ProxyBaseLLMRequestProcessing: For such a model, hooks other than guardrails and content enforcers run first and may rewrite the model. A model that still cannot route raises the routing error before any - guardrail runs; otherwise the guardrails run on the rewritten request. The registry - read-through is retried only for a model those hooks changed. + guardrail runs; otherwise the guardrails run on the rewritten request. A request those + hooks or the routing check reject is redacted. The registry read-through is retried only + for a model those hooks changed. """ if model_routable: self.data = await proxy_logging_obj.pre_call_hook( @@ -2326,14 +2327,14 @@ class ProxyBaseLLMRequestProcessing: ) return model_before_hooks: Final = self.data.get("model") - self.data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=self.data, - call_type=call_type, - skip_guardrails=True, - skip_content_enforcers=True, - ) try: + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + skip_guardrails=True, + skip_content_enforcers=True, + ) await raise_if_model_not_routable( data=self.data, llm_router=llm_router, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index f6a0bef00fb..0a923f9fd47 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -2,7 +2,7 @@ import asyncio import copy import datetime import json -from collections.abc import Mapping +from collections.abc import Awaitable, Mapping from types import MappingProxyType, SimpleNamespace from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence from urllib.parse import unquote_plus @@ -595,108 +595,6 @@ class TestProxyBaseLLMRequestProcessing: assert persisted_body[input_key][0]["content"] == "later input mutation" assert persisted_body["tools"][0]["description"] == "later tool mutation" - @pytest.mark.asyncio - async def test_common_processing_pre_call_logic_rejects_unroutable_model_before_pre_call_hook(self, monkeypatch): - from types import SimpleNamespace - - from litellm.proxy.common_utils import registry_read_through - from litellm.proxy.route_llm_request import ProxyModelNotFoundError - - processing_obj = ProxyBaseLLMRequestProcessing(data={}) - mock_request = MagicMock(spec=Request) - mock_request.headers = {} - - async def mock_add_litellm_data_to_request(*args, **kwargs): - return { - "model": "does-not-exist", - "messages": [{"role": "user", "content": "hi"}], - } - - monkeypatch.setattr( - litellm.proxy.common_request_processing, - "add_litellm_data_to_request", - mock_add_litellm_data_to_request, - ) - monkeypatch.setattr( - registry_read_through, - "model_registry_read_through", - SimpleNamespace(attempt=AsyncMock(return_value=False)), - ) - mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) - mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"]) - mock_proxy_config = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) - router = Router( - model_list=[ - { - "model_name": "gpt-4o-mini", - "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, - } - ] - ) - - with pytest.raises(ProxyModelNotFoundError) as exc_info: - await processing_obj.common_processing_pre_call_logic( - request=mock_request, - general_settings={}, - user_api_key_dict=ProxyUserAPIKeyAuth(), - proxy_logging_obj=mock_proxy_logging_obj, - proxy_config=mock_proxy_config, - route_type="acompletion", - llm_router=router, - ) - - assert exc_info.value.status_code == 400 - mock_proxy_logging_obj.pre_call_hook.assert_awaited_once() - hook_kwargs: Final = mock_proxy_logging_obj.pre_call_hook.await_args.kwargs - assert hook_kwargs["skip_guardrails"] is True - assert hook_kwargs["skip_content_enforcers"] is True - - @pytest.mark.asyncio - async def test_common_processing_pre_call_logic_runs_pre_call_hook_for_routable_model(self, monkeypatch): - processing_obj = ProxyBaseLLMRequestProcessing(data={}) - mock_request = MagicMock(spec=Request) - mock_request.headers = {} - - async def mock_add_litellm_data_to_request(*args, **kwargs): - return { - "model": "gpt-4o-mini", - "messages": [{"role": "user", "content": "hi"}], - } - - async def mock_pre_call_hook(user_api_key_dict, data, call_type, skip_guardrails=False): - return data - - monkeypatch.setattr( - litellm.proxy.common_request_processing, - "add_litellm_data_to_request", - mock_add_litellm_data_to_request, - ) - mock_proxy_logging_obj = MagicMock(spec=ProxyLogging) - mock_proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=mock_pre_call_hook) - mock_proxy_config = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) - router = Router( - model_list=[ - { - "model_name": "gpt-4o-mini", - "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, - } - ] - ) - - await processing_obj.common_processing_pre_call_logic( - request=mock_request, - general_settings={}, - user_api_key_dict=ProxyUserAPIKeyAuth(), - proxy_logging_obj=mock_proxy_logging_obj, - proxy_config=mock_proxy_config, - route_type="acompletion", - llm_router=router, - ) - - mock_proxy_logging_obj.pre_call_hook.assert_awaited_once() - @staticmethod def _guardrail_tag_budget_harness( monkeypatch, @@ -10792,6 +10690,11 @@ class _ModelRewritingHook(CustomLogger): return data +class _RejectingHook(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + raise HTTPException(status_code=403, detail="rejected by hook") + + class TestPreCallHooksForUnroutableModel: @staticmethod async def _run( @@ -10800,36 +10703,39 @@ class TestPreCallHooksForUnroutableModel: llm_router: Router | None, processor: ProxyBaseLLMRequestProcessing | None = None, skip_guardrails: bool = False, - read_through_attempt: AsyncMock | None = None, + model_db_lookup: Callable[[str], Awaitable[bool]] | None = None, extra_body: Mapping[str, object] | None = None, ) -> dict: + from litellm.proxy import proxy_server from litellm.proxy.common_utils import registry_read_through from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache - async def mock_add_litellm_data_to_request(*args, **kwargs): - messages: Final = [{"role": "user", "content": "email jane@example.com"}] - body: Final = {"model": model, "messages": messages, **copy.deepcopy(dict(extra_body or {}))} - return {**body, "proxy_server_request": {"body": copy.deepcopy(body)}} - - monkeypatch.setattr( - litellm.proxy.common_request_processing, "add_litellm_data_to_request", mock_add_litellm_data_to_request - ) monkeypatch.setattr( registry_read_through, "model_registry_read_through", - SimpleNamespace(attempt=read_through_attempt or AsyncMock(return_value=False)), + registry_read_through.RegistryReadThrough( + resync=model_db_lookup or registry_read_through._resync_model_deployments, + is_loaded=registry_read_through._model_is_loaded, + ), + ) + target: Final = processor or ProxyBaseLLMRequestProcessing(data={}) + target.data.update( + { + "model": model, + "messages": [{"role": "user", "content": "email jane@example.com"}], + **copy.deepcopy(dict(extra_body or {})), + } + ) + request: Final = Request( + {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""} ) - mock_request: Final = MagicMock(spec=Request) - mock_request.headers = {} - mock_proxy_config: Final = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache()) - data, _ = await (processor or ProxyBaseLLMRequestProcessing(data={})).common_processing_pre_call_logic( - request=mock_request, + data, _ = await target.common_processing_pre_call_logic( + request=request, general_settings={}, user_api_key_dict=ProxyUserAPIKeyAuth(), proxy_logging_obj=proxy_logging_obj, - proxy_config=mock_proxy_config, + proxy_config=proxy_server.ProxyConfig(), route_type="acompletion", llm_router=llm_router, skip_guardrails=skip_guardrails, @@ -10891,6 +10797,16 @@ class TestPreCallHooksForUnroutableModel: assert (hook.calls, guardrail.models_seen, enforcer.calls) == (1, ["gpt-4o-mini"], 1) + @pytest.mark.asyncio + async def test_routable_model_keeps_the_configured_hook_order(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail, _RejectingHook()]) + + with pytest.raises(HTTPException): + await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router()) + + assert guardrail.models_seen == ["gpt-4o-mini"] + @pytest.mark.asyncio async def test_fully_blocked_model_is_a_403_without_guardrails(self, monkeypatch): guardrail: Final = _CountingGuardrail() @@ -10947,6 +10863,22 @@ class TestPreCallHooksForUnroutableModel: assert processor.data["messages"] == [{"role": "user", "content": expected_content}] assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + @pytest.mark.asyncio + async def test_request_a_hook_rejects_before_guardrails_is_redacted(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [guardrail, _RejectingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor) + + assert exc_info.value.status_code == 403 + assert guardrail.models_seen == [] + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is True + @pytest.mark.asyncio @pytest.mark.parametrize( ("extra_body", "expected_body"), @@ -11007,12 +10939,16 @@ class TestPreCallHooksForUnroutableModel: from litellm.proxy.route_llm_request import ProxyModelNotFoundError monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()]) - attempt: Final = AsyncMock(return_value=False) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False with pytest.raises(ProxyModelNotFoundError): - await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), read_through_attempt=attempt) + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), model_db_lookup=no_db_row) - assert [call.args[0] for call in attempt.await_args_list] == ["does-not-exist"] + assert looked_up == ["does-not-exist"] @pytest.mark.asyncio async def test_model_rewritten_by_a_hook_is_read_through_under_its_new_name(self, monkeypatch): @@ -11023,12 +10959,16 @@ class TestPreCallHooksForUnroutableModel: "callbacks", [_ModelRewritingHook(source_model="unknown-virtual", target_model="db-only-model")], ) - attempt: Final = AsyncMock(return_value=False) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False with pytest.raises(ProxyModelNotFoundError): - await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), read_through_attempt=attempt) + await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), model_db_lookup=no_db_row) - assert [call.args[0] for call in attempt.await_args_list] == ["unknown-virtual", "db-only-model"] + assert looked_up == ["unknown-virtual", "db-only-model"] @pytest.mark.asyncio async def test_no_router_skips_the_routability_check(self, monkeypatch): diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index eacceaa5727..916a6a4944e 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1727,19 +1727,9 @@ def _router_with_gpt4o_mini(): @pytest.mark.asyncio -async def test_raise_if_model_not_routable_raises_for_unknown_model(monkeypatch): - from types import SimpleNamespace - from unittest.mock import AsyncMock - - from litellm.proxy.common_utils import registry_read_through +async def test_raise_if_model_not_routable_raises_for_unknown_model(): from litellm.proxy.route_llm_request import raise_if_model_not_routable - monkeypatch.setattr( - registry_read_through, - "model_registry_read_through", - SimpleNamespace(attempt=AsyncMock(return_value=False)), - ) - with pytest.raises(ProxyModelNotFoundError) as exc_info: await raise_if_model_not_routable( data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]}, @@ -1770,21 +1760,9 @@ async def test_raise_if_model_not_routable_raises_for_unknown_model(monkeypatch) ], ) @pytest.mark.asyncio -async def test_raise_if_model_not_routable_reports_body_errors_before_the_model( - monkeypatch, data, route_type, expected_error -): - from types import SimpleNamespace - from unittest.mock import AsyncMock - - from litellm.proxy.common_utils import registry_read_through +async def test_raise_if_model_not_routable_reports_body_errors_before_the_model(data, route_type, expected_error): from litellm.proxy.route_llm_request import raise_if_model_not_routable - monkeypatch.setattr( - registry_read_through, - "model_registry_read_through", - SimpleNamespace(attempt=AsyncMock(return_value=False)), - ) - with pytest.raises(expected_error): await raise_if_model_not_routable( data=data, @@ -1831,18 +1809,8 @@ async def test_raise_if_model_not_routable_reports_body_errors_before_the_model( ) @pytest.mark.asyncio async def test_raise_if_model_not_routable_allows_routable_requests(monkeypatch, data, route_type, router_mutator): - from types import SimpleNamespace - from unittest.mock import AsyncMock - - from litellm.proxy.common_utils import registry_read_through from litellm.proxy.route_llm_request import raise_if_model_not_routable - monkeypatch.setattr( - registry_read_through, - "model_registry_read_through", - SimpleNamespace(attempt=AsyncMock(return_value=False)), - ) - if router_mutator == "wildcard": import litellm From a7d7e82a88a471ba14cee3097159857eba58a3f6 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:46:11 -0700 Subject: [PATCH 13/24] refactor(proxy): reuse route_request's dispatch decision for the pre-guardrail model check Split _route_request_single_attempt into _decide_dispatch, which picks where the request goes without changing it, and the send step. The pre-call check now asks that same decision instead of a copied predicate, checks only loaded models before the hooks, and looks a model up in the DB once after them. A request rejected there keeps the snapshot the hooks produced in its logs --- litellm/proxy/common_request_processing.py | 16 +- litellm/proxy/route_llm_request.py | 744 ++++++++---------- .../proxy/test_common_request_processing.py | 53 +- 3 files changed, 383 insertions(+), 430 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index dd2fae48a6c..153f85881c2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -120,7 +120,7 @@ from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( - is_model_routable, + can_route_with_loaded_models, raise_if_model_not_routable, route_request, ) @@ -1813,7 +1813,7 @@ async def _team_member_within_fallback_model_budget( class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): - self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place + self.data = data self.served_model = served_model self._tags_before_guardrails: frozenset[str] | None = None @@ -2335,8 +2335,8 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) - model_routable: Final = llm_router is None or await is_model_routable( - data=self.data, # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # loosely typed + model_routable: Final = llm_router is None or can_route_with_loaded_models( + data=self.data, llm_router=llm_router, user_model=user_model, route_type=route_type, @@ -2401,9 +2401,8 @@ class ProxyBaseLLMRequestProcessing: For such a model, hooks other than guardrails and content enforcers run first and may rewrite the model. A model that still cannot route raises the routing error before any - guardrail runs; otherwise the guardrails run on the rewritten request. A request those - hooks or the routing check reject is redacted. The registry read-through is retried only - for a model those hooks changed. + guardrail runs, otherwise the guardrails run on the rewritten request. A request those + hooks or the routing check reject is redacted. """ if model_routable: self.data = await proxy_logging_obj.pre_call_hook( @@ -2413,7 +2412,6 @@ class ProxyBaseLLMRequestProcessing: skip_guardrails=skip_guardrails, ) return - model_before_hooks: Final = self.data.get("model") try: self.data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -2422,12 +2420,12 @@ class ProxyBaseLLMRequestProcessing: skip_guardrails=True, skip_content_enforcers=True, ) + refresh_proxy_server_request_body_snapshot(self.data) await raise_if_model_not_routable( data=self.data, llm_router=llm_router, user_model=user_model, route_type=call_type, - read_through=self.data.get("model") != model_before_hooks, ) except Exception: if not skip_guardrails: diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 58b5170e1d8..cbd0e872faf 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -1,12 +1,14 @@ import asyncio -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass +from enum import Enum, auto from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, Literal +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar import httpx from fastapi import HTTPException, status from pydantic import TypeAdapter, ValidationError +from typing_extensions import Never import litellm from litellm.proxy._types import ProxyException, UserAPIKeyAuth @@ -37,6 +39,8 @@ if TYPE_CHECKING: else: LitellmRouter = Any +_T = TypeVar("_T") + def _route_user_config_request(data: dict, route_type: str): """Route a request using the user-provided router config.""" @@ -153,143 +157,6 @@ ROUTE_ENDPOINT_MAPPING: Final = { _AVAILABLE_MODELS_HINT: Final = "Call `/v1/models` to view available models for your key." -_EVAL_ROUTE_TYPES: Final = frozenset( - { - "acreate_eval", - "alist_evals", - "aget_eval", - "aupdate_eval", - "adelete_eval", - "acancel_eval", - "acreate_run", - "alist_runs", - "aget_run", - "acancel_run", - "adelete_run", - } -) - -_CONTAINER_ROUTE_TYPES: Final = frozenset( - { - "acreate_container", - "alist_containers", - "aretrieve_container", - "adelete_container", - "aupload_container_file", - "alist_container_files", - "aretrieve_container_file", - "adelete_container_file", - "aretrieve_container_file_content", - } -) - -_INTERACTION_ROUTE_TYPES: Final = frozenset( - { - "acreate_interaction", - "aget_interaction", - "adelete_interaction", - "acancel_interaction", - } -) - -_MANAGED_AGENT_ROUTE_TYPES: Final = frozenset( - { - "acreate_agent", - "alist_agents", - "aget_agent", - "adelete_agent", - "alist_agent_versions", - } -) - -_MODEL_OPTIONAL_ROUTE_TYPES: Final = frozenset( - { - "avideo_list", - "avideo_status", - "avideo_content", - "avideo_remix", - "avideo_create_character", - "avideo_get_character", - "avideo_edit", - "avideo_extension", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_content", - "avector_store_file_delete", - "acreate_skill", - "alist_skills", - "aget_skill", - "adelete_skill", - "aingest", - } -) - -_MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES: Final = frozenset( - { - "amoderation", - "aget_responses", - "adelete_responses", - "acancel_responses", - "alist_input_items", - "avector_store_create", - "avector_store_search", - "avector_store_retrieve", - "avector_store_list", - "avector_store_update", - "avector_store_delete", - "avector_store_file_create", - "avector_store_file_list", - "avector_store_file_retrieve", - "avector_store_file_content", - "avector_store_file_update", - "avector_store_file_delete", - "asearch", - "acreate_container", - "alist_containers", - "aretrieve_container", - "adelete_container", - "aupload_container_file", - "alist_container_files", - "aretrieve_container_file", - "adelete_container_file", - "aretrieve_container_file_content", - } -) - -_VIDEO_FALLBACK_ROUTE_TYPES: Final = frozenset( - { - "avideo_status", - "avideo_content", - "avideo_remix", - "avideo_create_character", - "avideo_get_character", - "avideo_edit", - "avideo_extension", - } -) - -_NO_MODEL_ROUTING_ROUTE_TYPES: Final = frozenset( - { - "apply_guardrail", - "call_mcp_tool", - # Files, fine-tuning, batch, and A2A endpoints resolve their own target - # (file id, job id, or agent) after pre-call processing and never reach - # _route_request_single_attempt. - "afile_content", - "afile_retrieve", - "afile_delete", - "acreate_fine_tuning_job", - "aretrieve_fine_tuning_job", - "alist_fine_tuning_jobs", - "acancel_fine_tuning_job", - "acreate_batch", - "aretrieve_batch", - "alist_batches", - "acancel_batch", - "asend_message", - } -) - class ProxyModelNotFoundError(HTTPException): def __init__(self, route: str, model_name: str, retryable_with_model_read_through: bool = True): @@ -743,14 +610,46 @@ async def route_request( """ Common helper to route the request """ - try: - return await _route_request_single_attempt( + return await _retry_once_after_model_read_through( + data, + llm_router, + lambda router: _route_request_single_attempt( data=data, - llm_router=llm_router, + llm_router=router, user_model=user_model, route_type=route_type, user_api_key_dict=user_api_key_dict, - ) + ), + ) + + +async def raise_if_model_not_routable( + data: dict, # mutable-ok: route_request's untyped request body, read here and never changed + llm_router: LitellmRouter | None, + user_model: str | None, + route_type: str, +) -> None: + """Raise the error route_request would raise before sending this request, without sending it.""" + + async def check(router: LitellmRouter | None) -> None: + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router) + raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) + if ( + _decide_dispatch(data=data, llm_router=router, user_model=user_model, route_type=route_type) + is _Dispatch.MODEL_NOT_FOUND + ): + _raise_model_not_found(data=data, route_type=route_type) + + await _retry_once_after_model_read_through(data, llm_router, check) + + +async def _retry_once_after_model_read_through( + data: Mapping[str, object], + llm_router: LitellmRouter | None, + attempt: Callable[[LitellmRouter | None], Awaitable[_T]], +) -> _T: + try: + return await attempt(llm_router) except (ProxyModelNotFoundError, ProxyMissingParamWithoutLoadedModelError) as e: requested_model: Final = data.get("model", "") if ( @@ -766,154 +665,236 @@ async def route_request( if not await model_registry_read_through.attempt(requested_model): raise - return await _route_request_single_attempt( - data=data, - llm_router=proxy_server.llm_router, - user_model=user_model, - route_type=route_type, - user_api_key_dict=user_api_key_dict, - ) + return await attempt(proxy_server.llm_router) -def _router_can_route( - data: Mapping[str, object], +def _raise_model_not_found(data: Mapping[str, object], route_type: str) -> Never: + # if no route found then it's a bad request + route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) + raise ProxyModelNotFoundError( + route=route_name, + model_name=str(data.get("model", "")), + ) + + +class _Dispatch(Enum): + ROUTER = auto() + ROUTER_SPECIFIC_DEPLOYMENT = auto() + ROUTER_THEN_LITELLM = auto() + LITELLM = auto() + BATCH_COMPLETION = auto() + USER_CONFIG = auto() + ROUTER_SETTINGS_OVERRIDE = auto() + EVAL = auto() + A2A_AGENT = auto() + MISSING_MODEL = auto() + MODEL_NOT_FOUND = auto() + + +@dataclass(frozen=True, slots=True) +class _TeamModel: + name: str + + +def _decide_dispatch( + data: dict, # mutable-ok: route_request's untyped request body, read here and never changed llm_router: LitellmRouter | None, user_model: str | None, route_type: str, -) -> bool: - """Pure predicate mirroring the branch structure of _route_request_single_attempt. +) -> _Dispatch | _TeamModel: + """Where route_request sends this request, decided without changing it. - Returns True when the existing routing tree would dispatch this request - somewhere instead of falling through to the final ProxyModelNotFoundError. + Raises PermissionDeniedError (403) when every deployment of the requested model is blocked. """ - if route_type in _NO_MODEL_ROUTING_ROUTE_TYPES: - return True - if "api_key" in data or "api_base" in data: - return True - model: Final = data.get("model") - if route_type == "acompletion" and isinstance(model, str) and "," in model and llm_router is not None: - return True - if "user_config" in data or "router_settings_override" in data: - return True - if llm_router is None: - return user_model is not None or route_type == "allm_passthrough_route" - _raise_if_model_fully_blocked(llm_router=llm_router, model_name=model, team_id=get_team_id_from_data(dict(data))) - if ( - route_type in _EVAL_ROUTE_TYPES - or route_type in _CONTAINER_ROUTE_TYPES - or route_type in _INTERACTION_ROUTE_TYPES - or route_type in _MANAGED_AGENT_ROUTE_TYPES - ): - return True - if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and (model is None or model == ""): - return True - if not isinstance(model, str): - return True - - return _router_model_group_routable(model=model, llm_router=llm_router, data=data, route_type=route_type) - - -def _router_model_group_routable( - model: str, - llm_router: LitellmRouter, - data: Mapping[str, object], - route_type: str, -) -> bool: - team_id: Final = get_team_id_from_data(dict(data)) - if team_id is not None and llm_router.map_team_model(model, team_id) is not None: - return True - + team_id: Final = get_team_id_from_data(data) + router_model_names: Final = llm_router.model_names if llm_router is not None else [] is_proxy_admin_without_team: Final = team_id is None and is_proxy_admin_request(data) - if ( - is_proxy_admin_without_team - and model not in llm_router.model_names # pyright: ignore[reportUnknownMemberType] # bare set - and model in llm_router.team_public_model_names - ) or llm_router.is_recognized_model(model): - return True - if model in llm_router.model_names: # pyright: ignore[reportUnknownMemberType] # bare set - return False - if llm_router.router_general_settings.pass_through_all_models: - return True - if llm_router.default_deployment is not None: - return True - if llm_router.pattern_router.patterns: # pyright: ignore[reportUnknownMemberType] # untyped dict - return True - if model in llm_router.deployment_names: # pyright: ignore[reportUnknownMemberType] # bare list - return True - if route_type in _MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES or route_type in _VIDEO_FALLBACK_ROUTE_TYPES: - return True - return _is_a2a_agent_model(model) + if "api_key" in data or "api_base" in data: + if llm_router is not None: + return _Dispatch.ROUTER + else: + return _Dispatch.LITELLM + + elif ( + route_type == "acompletion" + and data.get("model", "") is not None + and "," in data.get("model", "") + and llm_router is not None + ): + # Handle batch completions with comma-separated models BEFORE user_config check + # This ensures batch completion logic is applied even when user_config is set + return _Dispatch.BATCH_COMPLETION + + elif data.get("user_config") is not None: + return _Dispatch.USER_CONFIG + + elif "router_settings_override" in data: + return _Dispatch.ROUTER_SETTINGS_OVERRIDE + elif llm_router is not None: + _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) + # Evals API: always route to litellm directly (not through router) + # But extract model credentials if a model is provided + if route_type in [ + "acreate_eval", + "alist_evals", + "aget_eval", + "aupdate_eval", + "adelete_eval", + "acancel_eval", + "acreate_run", + "alist_runs", + "aget_run", + "acancel_run", + "adelete_run", + ]: + return _Dispatch.EVAL + # Skip model-based routing for container operations + if route_type in [ + "acreate_container", + "alist_containers", + "aretrieve_container", + "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + ]: + return _Dispatch.ROUTER + # Interactions API: create with agent, get/delete/cancel don't need model routing + if route_type in [ + "acreate_interaction", + "aget_interaction", + "adelete_interaction", + "acancel_interaction", + ]: + return _Dispatch.ROUTER + # Managed Agents API: these don't need model routing + if route_type in [ + "acreate_agent", + "alist_agents", + "aget_agent", + "adelete_agent", + "alist_agent_versions", + ]: + return _Dispatch.ROUTER + if route_type in [ + "avideo_list", + "avideo_status", + "avideo_content", + "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_content", + "avector_store_file_delete", + "acreate_skill", + "alist_skills", + "aget_skill", + "adelete_skill", + "aingest", + ] and (data.get("model") is None or data.get("model") == ""): + # These endpoints don't need a model, use custom_llm_provider directly + return _Dispatch.LITELLM + + if "model" not in data: + return _Dispatch.MISSING_MODEL + team_model_name: Final = llm_router.map_team_model(data["model"], team_id) if team_id is not None else None + if team_model_name is not None: + return _TeamModel(name=team_model_name) + + elif ( + is_proxy_admin_without_team + and data["model"] not in router_model_names + and data["model"] in llm_router.team_public_model_names + ) or llm_router.is_recognized_model(data["model"]): + return _Dispatch.ROUTER + + elif data["model"] not in router_model_names: + # Check wildcards before checking deployment_names + # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match + if llm_router.router_general_settings.pass_through_all_models: + return _Dispatch.LITELLM + elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: + return _Dispatch.ROUTER + elif data["model"] in llm_router.deployment_names: + # Only match deployment_names if no wildcard matched + return _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT + elif route_type in [ + "amoderation", + "aget_responses", + "adelete_responses", + "acancel_responses", + "alist_input_items", + "avector_store_create", + "avector_store_search", + "avector_store_retrieve", + "avector_store_list", + "avector_store_update", + "avector_store_delete", + "avector_store_file_create", + "avector_store_file_list", + "avector_store_file_retrieve", + "avector_store_file_content", + "avector_store_file_update", + "avector_store_file_delete", + "asearch", + "acreate_container", + "alist_containers", + "aretrieve_container", + "adelete_container", + "aupload_container_file", + "alist_container_files", + "aretrieve_container_file", + "adelete_container_file", + "aretrieve_container_file_content", + ]: + # These endpoints can work with or without model parameter + return _Dispatch.ROUTER + elif route_type in [ + "avideo_status", + "avideo_content", + "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", + ]: + # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), + # try router first to allow for multi-deployment load balancing + return _Dispatch.ROUTER_THEN_LITELLM + elif _is_a2a_agent_model(data.get("model", "")): + return _Dispatch.A2A_AGENT + + elif user_model is not None or route_type == "allm_passthrough_route": + return _Dispatch.LITELLM + + return _Dispatch.MODEL_NOT_FOUND -def _router_can_route_unless_blocked( - data: Mapping[str, object], +def can_route_with_loaded_models( + data: dict, # mutable-ok: route_request's untyped request body, read here and never changed llm_router: LitellmRouter | None, user_model: str | None, route_type: str, ) -> bool: + """Whether route_request would send this request somewhere, using only the models this worker has loaded. + + A fully blocked model counts as not routable. + """ try: - return _router_can_route(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type) + return ( + _decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type) + is not _Dispatch.MODEL_NOT_FOUND + ) except litellm.PermissionDeniedError: return False -async def is_model_routable( - data: Mapping[str, object], - llm_router: LitellmRouter | None, - user_model: str | None, - route_type: str, - read_through: bool = True, -) -> bool: - """Whether route_request would dispatch this request, including its registry read-through retry. - - A fully blocked model counts as not routable. read_through=False skips the registry retry. - """ - if _router_can_route_unless_blocked(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): - return True - requested_model: Final = data.get("model") - if not read_through or not isinstance(requested_model, str) or not requested_model: - return False - from litellm.proxy import proxy_server - from litellm.proxy.common_utils.registry_read_through import ( - model_registry_read_through, - ) - - return await model_registry_read_through.attempt(requested_model) and _router_can_route_unless_blocked( - data=data, llm_router=proxy_server.llm_router, user_model=user_model, route_type=route_type - ) - - -async def raise_if_model_not_routable( - data: Mapping[str, object], - llm_router: LitellmRouter | None, - user_model: str | None, - route_type: str, - read_through: bool = True, -) -> None: - """Raise the error route_request would raise for a model it cannot dispatch. - - The body checks route_request runs first come first: a missing required param, then - disallowed mock testing params. After them, PermissionDeniedError (403) for a fully blocked - model and ProxyModelNotFoundError (400) otherwise. read_through=False skips the registry retry. - """ - if await is_model_routable( - data=data, llm_router=llm_router, user_model=user_model, route_type=route_type, read_through=read_through - ): - return - raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router) - raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed()) - requested_model: Final = data.get("model") - if llm_router is not None: - _raise_if_model_fully_blocked( - llm_router=llm_router, model_name=requested_model, team_id=get_team_id_from_data(dict(data)) - ) - raise ProxyModelNotFoundError( - route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), - model_name=requested_model if isinstance(requested_model, str) else "", - ) - - async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request llm_router: LitellmRouter | None, @@ -931,162 +912,103 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr if "user_config" in data and data["user_config"] is None: data.pop("user_config") # pyright: ignore[reportUnknownMemberType] # route_request takes an untyped dict body - team_id: Final = get_team_id_from_data(data) - router_model_names: Final = llm_router.model_names if llm_router is not None else [] - is_proxy_admin_without_team: Final = team_id is None and is_proxy_admin_request(data) - # Preprocess Google GenAI generate content requests if route_type in ["agenerate_content", "agenerate_content_stream"]: # Map generationConfig to config parameter for Google GenAI compatibility if "generationConfig" in data and "config" not in data: data["config"] = data.pop("generationConfig") - if "api_key" in data or "api_base" in data: - if llm_router is not None: - return getattr(llm_router, f"{route_type}")(**data) - else: - return getattr(litellm, f"{route_type}")(**data) - elif ( - route_type == "acompletion" - and data.get("model", "") is not None - and "," in data.get("model", "") - and llm_router is not None - ): - # Handle batch completions with comma-separated models BEFORE user_config check - # This ensures batch completion logic is applied even when user_config is set - if data.get("fastest_response", False): - return llm_router.abatch_completion_fastest_response(**data) - else: + match _decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type): + case _Dispatch.ROUTER: + return getattr(llm_router, f"{route_type}")(**data) + case _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT: + return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True) + case _Dispatch.ROUTER_THEN_LITELLM: + try: + return getattr(llm_router, f"{route_type}")(**data) + except Exception: + # If router fails (e.g., model not found in router), fall back to direct call + return getattr(litellm, f"{route_type}")(**data) + case _Dispatch.LITELLM: + return getattr(litellm, f"{route_type}")(**data) + case _Dispatch.BATCH_COMPLETION if llm_router is not None: + if data.get("fastest_response", False): + return llm_router.abatch_completion_fastest_response(**data) models: Final = [model.strip() for model in data.pop("model").split(",")] return llm_router.abatch_completion(models=models, **data) - - elif "user_config" in data: - return _route_user_config_request(data, route_type) - - elif "router_settings_override" in data: - # Apply per-request router settings overrides from key/team config - # Instead of creating a new Router (expensive), merge settings into kwargs - # The Router already supports per-request overrides for these settings - override_settings: Final = data.pop("router_settings_override") - - # Settings that the Router accepts as per-request kwargs - # These override the global router settings for this specific request - per_request_settings: Final = [ - "fallbacks", - "context_window_fallbacks", - "content_policy_fallbacks", - "num_retries", - "timeout", - "model_group_retry_policy", - "routing_strategy", - "enable_tag_filtering", - ] - - for key in per_request_settings: - if override_settings.get(key) is not None and key not in data: - data[key] = override_settings[key] - - # Use main router with overridden kwargs - if llm_router is not None: - return getattr(llm_router, f"{route_type}")(**data) - else: + case _Dispatch.USER_CONFIG: + return _route_user_config_request(data, route_type) + case _Dispatch.ROUTER_SETTINGS_OVERRIDE: + _merge_router_settings_override(data) + return getattr(llm_router if llm_router is not None else litellm, f"{route_type}")(**data) + case _Dispatch.EVAL: + _merge_eval_model_credentials(data, llm_router) return getattr(litellm, f"{route_type}")(**data) - elif llm_router is not None: - _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) - # Evals API: always route to litellm directly (not through router) - # But extract model credentials if a model is provided - if route_type in _EVAL_ROUTE_TYPES: - # If a model is provided, get its credentials from the router - model: Final = data.get("model") - if model and llm_router: - try: - # Try to get deployment credentials for this model - deployment_creds = llm_router.get_deployment_credentials(model_id=model) - if not deployment_creds: - # Try by model group name - deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) - if ( - deployment - and deployment.litellm_params - and not llm_router.is_deployment_blocked(deployment) - ): - deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) - - # If we found credentials, merge them into data (but don't override user-provided values) - if deployment_creds: - data.update(deployment_creds) - except Exception: - # If we can't get deployment creds, continue without them - pass - - return getattr(litellm, f"{route_type}")(**data) - # Skip model-based routing for container operations - if route_type in _CONTAINER_ROUTE_TYPES: + case _TeamModel(name=team_model_name): + data["model"] = team_model_name return getattr(llm_router, f"{route_type}")(**data) - # Interactions API: create with agent, get/delete/cancel don't need model routing - if route_type in _INTERACTION_ROUTE_TYPES: - return getattr(llm_router, f"{route_type}")(**data) - # Managed Agents API: these don't need model routing - if route_type in _MANAGED_AGENT_ROUTE_TYPES: - return getattr(llm_router, f"{route_type}")(**data) - if route_type in _MODEL_OPTIONAL_ROUTE_TYPES and (data.get("model") is None or data.get("model") == ""): - # These endpoints don't need a model, use custom_llm_provider directly - return getattr(litellm, f"{route_type}")(**data) - - if "model" not in data: + case _Dispatch.MISSING_MODEL: raise ProxyMissingRequiredParamError( route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type), param="model", ) - team_model_name: Final = llm_router.map_team_model(data["model"], team_id) if team_id is not None else None - if team_model_name is not None: - data["model"] = team_model_name - return getattr(llm_router, f"{route_type}")(**data) + case _Dispatch.A2A_AGENT: + from litellm.proxy.agent_endpoints.a2a_routing import ( + route_a2a_agent_request, + ) - elif ( - is_proxy_admin_without_team - and data["model"] not in router_model_names - and data["model"] in llm_router.team_public_model_names - ) or llm_router.is_recognized_model(data["model"]): - return getattr(llm_router, f"{route_type}")(**data) + result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict) + if result is not None: + return result - elif data["model"] not in router_model_names: - # Check wildcards before checking deployment_names - # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match - if llm_router.router_general_settings.pass_through_all_models: - return getattr(litellm, f"{route_type}")(**data) - elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: - return getattr(llm_router, f"{route_type}")(**data) - elif data["model"] in llm_router.deployment_names: - # Only match deployment_names if no wildcard matched - return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True) - elif route_type in _MODEL_OPTIONAL_FALLBACK_ROUTE_TYPES: - # These endpoints can work with or without model parameter - return getattr(llm_router, f"{route_type}")(**data) - elif route_type in _VIDEO_FALLBACK_ROUTE_TYPES: - # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), - # try router first to allow for multi-deployment load balancing - try: - return getattr(llm_router, f"{route_type}")(**data) - except Exception: - # If router fails (e.g., model not found in router), fall back to direct call - return getattr(litellm, f"{route_type}")(**data) - elif _is_a2a_agent_model(data.get("model", "")): - from litellm.proxy.agent_endpoints.a2a_routing import ( - route_a2a_agent_request, - ) + _raise_model_not_found(data=data, route_type=route_type) - result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict) - if result is not None: - return result - # Fall through to raise exception below if result is None - elif user_model is not None or route_type == "allm_passthrough_route": - return getattr(litellm, f"{route_type}")(**data) +def _merge_router_settings_override( + data: dict, # mutable-ok: merges into the request body route_request dispatches +) -> None: + # Apply per-request router settings overrides from key/team config + # Instead of creating a new Router (expensive), merge settings into kwargs + # The Router already supports per-request overrides for these settings + override_settings: Final = data.pop("router_settings_override") - # if no route found then it's a bad request - route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type) - raise ProxyModelNotFoundError( - route=route_name, - model_name=data.get("model", ""), - ) + # Settings that the Router accepts as per-request kwargs + # These override the global router settings for this specific request + per_request_settings: Final = [ + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "timeout", + "model_group_retry_policy", + "routing_strategy", + "enable_tag_filtering", + ] + + for key in per_request_settings: + if override_settings.get(key) is not None and key not in data: + data[key] = override_settings[key] + + +def _merge_eval_model_credentials( + data: dict, # mutable-ok: merges into the request body route_request dispatches + llm_router: LitellmRouter | None, +) -> None: + # If a model is provided, get its credentials from the router + model: Final = data.get("model") + if model and llm_router: + try: + # Try to get deployment credentials for this model + deployment_creds = llm_router.get_deployment_credentials(model_id=model) + if not deployment_creds: + # Try by model group name + deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) + if deployment and deployment.litellm_params and not llm_router.is_deployment_blocked(deployment): + deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) + + # If we found credentials, merge them into data (but don't override user-provided values) + if deployment_creds: + data.update(deployment_creds) + except Exception: + # If we can't get deployment creds, continue without them + pass diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 33cc7193863..538db9c2cab 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -454,7 +454,6 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) route_type = "acompletion" # Call the actual method. @@ -468,7 +467,6 @@ class TestProxyBaseLLMRequestProcessing: proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, route_type=route_type, - llm_router=self._router_with_free_and_paid_models(), ) mock_proxy_logging_obj.pre_call_hook.assert_called_once() @@ -572,17 +570,14 @@ class TestProxyBaseLLMRequestProcessing: "add_litellm_data_to_request", AsyncMock(return_value={**raw_body, metadata_key: {}, "proxy_server_request": {"body": raw_body}}), ) - mock_proxy_config = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) returned_data, logging_obj = await processing_obj.common_processing_pre_call_logic( request=mock_request, general_settings={}, user_api_key_dict=MagicMock(spec=UserAPIKeyAuth), proxy_logging_obj=mock_proxy_logging_obj, - proxy_config=mock_proxy_config, + proxy_config=MagicMock(spec=ProxyConfig), route_type=route_type, - llm_router=self._router_with_free_and_paid_models(), ) proxy_request: Final = returned_data["proxy_server_request"] @@ -2026,7 +2021,6 @@ class TestProxyBaseLLMRequestProcessing: mock_general_settings = {} mock_user_api_key_dict = MagicMock(spec=UserAPIKeyAuth) mock_proxy_config = MagicMock(spec=ProxyConfig) - mock_proxy_config._get_hierarchical_router_settings = AsyncMock(return_value=None) route_type = "acompletion" ( @@ -2039,7 +2033,6 @@ class TestProxyBaseLLMRequestProcessing: proxy_logging_obj=mock_proxy_logging_obj, proxy_config=mock_proxy_config, route_type=route_type, - llm_router=self._router_with_free_and_paid_models(), ) # Verify queue_time_seconds is set and non-negative. Ends at start_time @@ -10940,6 +10933,15 @@ class _ModelRewritingHook(CustomLogger): return data +class _EmailMaskingHook(CustomLogger): + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + data["messages"] = [ + {**message, "content": message["content"].replace("jane@example.com", "[EMAIL]")} + for message in data["messages"] + ] + return data + + class _RejectingHook(CustomLogger): async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): raise HTTPException(status_code=403, detail="rejected by hook") @@ -11201,7 +11203,7 @@ class TestPreCallHooksForUnroutableModel: assert looked_up == ["does-not-exist"] @pytest.mark.asyncio - async def test_model_rewritten_by_a_hook_is_read_through_under_its_new_name(self, monkeypatch): + async def test_model_rewritten_by_a_hook_is_looked_up_once_under_its_new_name(self, monkeypatch): from litellm.proxy.route_llm_request import ProxyModelNotFoundError monkeypatch.setattr( @@ -11218,7 +11220,38 @@ class TestPreCallHooksForUnroutableModel: with pytest.raises(ProxyModelNotFoundError): await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), model_db_lookup=no_db_row) - assert looked_up == ["unknown-virtual", "db-only-model"] + assert looked_up == ["db-only-model"] + + @pytest.mark.asyncio + async def test_virtual_model_a_hook_maps_to_a_loaded_model_needs_no_db_lookup(self, monkeypatch): + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook()]) + looked_up: Final = list[str]() + + async def no_db_row(model_name: str) -> bool: + looked_up.append(model_name) + return False + + data: Final = await self._run( + monkeypatch, model="virtual-model", llm_router=self._router(), model_db_lookup=no_db_row + ) + + assert data["model"] == "gpt-4o-mini" + assert looked_up == [] + + @pytest.mark.asyncio + async def test_unknown_model_without_guardrails_logs_the_request_the_hooks_produced(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_EmailMaskingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert snapshot["messages"] == [{"role": "user", "content": "email [EMAIL]"}] + assert snapshot["litellm_call_id"] == processor.data["litellm_call_id"] + assert snapshot["metadata"] is processor.data["metadata"] @pytest.mark.asyncio async def test_no_router_skips_the_routability_check(self, monkeypatch): From 48a0d87096ab5f8b7dd31cd4fce702c9b3c9bb42 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:51:40 -0700 Subject: [PATCH 14/24] chore(proxy): type the dispatch decision and keep the send step inline --- litellm/proxy/common_request_processing.py | 2 +- litellm/proxy/route_llm_request.py | 154 +++++++++++---------- 2 files changed, 79 insertions(+), 77 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 153f85881c2..ba428944136 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1813,7 +1813,7 @@ async def _team_member_within_fallback_model_budget( class ProxyBaseLLMRequestProcessing: def __init__(self, data: dict, served_model: str | None = None): - self.data = data + self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place self.served_model = served_model self._tags_before_guardrails: frozenset[str] | None = None diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index cbd0e872faf..7bfb8077990 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -39,7 +39,7 @@ if TYPE_CHECKING: else: LitellmRouter = Any -_T = TypeVar("_T") +_T: Final = TypeVar("_T") def _route_user_config_request(data: dict, route_type: str): @@ -624,7 +624,7 @@ async def route_request( async def raise_if_model_not_routable( - data: dict, # mutable-ok: route_request's untyped request body, read here and never changed + data: Mapping[str, object], llm_router: LitellmRouter | None, user_model: str | None, route_type: str, @@ -696,8 +696,8 @@ class _TeamModel: name: str -def _decide_dispatch( - data: dict, # mutable-ok: route_request's untyped request body, read here and never changed +def _decide_dispatch( # noqa: C901 # one branch per dispatch target, moved out of _route_request_single_attempt + data: Mapping[str, object], llm_router: LitellmRouter | None, user_model: str | None, route_type: str, @@ -706,7 +706,8 @@ def _decide_dispatch( Raises PermissionDeniedError (403) when every deployment of the requested model is blocked. """ - team_id: Final = get_team_id_from_data(data) + model: Final = data.get("model") + team_id: Final = get_team_id_from_data(dict(data)) router_model_names: Final = llm_router.model_names if llm_router is not None else [] is_proxy_admin_without_team: Final = team_id is None and is_proxy_admin_request(data) @@ -716,12 +717,7 @@ def _decide_dispatch( else: return _Dispatch.LITELLM - elif ( - route_type == "acompletion" - and data.get("model", "") is not None - and "," in data.get("model", "") - and llm_router is not None - ): + elif route_type == "acompletion" and isinstance(model, str) and "," in model and llm_router is not None: # Handle batch completions with comma-separated models BEFORE user_config check # This ensures batch completion logic is applied even when user_config is set return _Dispatch.BATCH_COMPLETION @@ -732,7 +728,7 @@ def _decide_dispatch( elif "router_settings_override" in data: return _Dispatch.ROUTER_SETTINGS_OVERRIDE elif llm_router is not None: - _raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id) + _raise_if_model_fully_blocked(llm_router=llm_router, model_name=model, team_id=team_id) # Evals API: always route to litellm directly (not through router) # But extract model credentials if a model is provided if route_type in [ @@ -797,31 +793,38 @@ def _decide_dispatch( "aget_skill", "adelete_skill", "aingest", - ] and (data.get("model") is None or data.get("model") == ""): + ] and (model is None or model == ""): # These endpoints don't need a model, use custom_llm_provider directly return _Dispatch.LITELLM if "model" not in data: return _Dispatch.MISSING_MODEL - team_model_name: Final = llm_router.map_team_model(data["model"], team_id) if team_id is not None else None + team_model_name: Final = ( + llm_router.map_team_model(model, team_id) + if team_id is not None and (model is None or isinstance(model, str)) + else None + ) if team_model_name is not None: return _TeamModel(name=team_model_name) - elif ( - is_proxy_admin_without_team - and data["model"] not in router_model_names - and data["model"] in llm_router.team_public_model_names - ) or llm_router.is_recognized_model(data["model"]): + elif isinstance(model, str) and ( + ( + is_proxy_admin_without_team + and model not in router_model_names + and model in llm_router.team_public_model_names + ) + or llm_router.is_recognized_model(model) + ): return _Dispatch.ROUTER - elif data["model"] not in router_model_names: + elif model not in router_model_names: # Check wildcards before checking deployment_names # Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match if llm_router.router_general_settings.pass_through_all_models: return _Dispatch.LITELLM elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0: return _Dispatch.ROUTER - elif data["model"] in llm_router.deployment_names: + elif model in llm_router.deployment_names: # Only match deployment_names if no wildcard matched return _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT elif route_type in [ @@ -867,7 +870,7 @@ def _decide_dispatch( # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), # try router first to allow for multi-deployment load balancing return _Dispatch.ROUTER_THEN_LITELLM - elif _is_a2a_agent_model(data.get("model", "")): + elif _is_a2a_agent_model(model): return _Dispatch.A2A_AGENT elif user_model is not None or route_type == "allm_passthrough_route": @@ -877,7 +880,7 @@ def _decide_dispatch( def can_route_with_loaded_models( - data: dict, # mutable-ok: route_request's untyped request body, read here and never changed + data: Mapping[str, object], llm_router: LitellmRouter | None, user_model: str | None, route_type: str, @@ -939,10 +942,57 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr case _Dispatch.USER_CONFIG: return _route_user_config_request(data, route_type) case _Dispatch.ROUTER_SETTINGS_OVERRIDE: - _merge_router_settings_override(data) - return getattr(llm_router if llm_router is not None else litellm, f"{route_type}")(**data) + # Apply per-request router settings overrides from key/team config + # Instead of creating a new Router (expensive), merge settings into kwargs + # The Router already supports per-request overrides for these settings + override_settings: Final = data.pop("router_settings_override") + + # Settings that the Router accepts as per-request kwargs + # These override the global router settings for this specific request + per_request_settings: Final = [ + "fallbacks", + "context_window_fallbacks", + "content_policy_fallbacks", + "num_retries", + "timeout", + "model_group_retry_policy", + "routing_strategy", + "enable_tag_filtering", + ] + + for key in per_request_settings: + if override_settings.get(key) is not None and key not in data: + data[key] = override_settings[key] + + # Use main router with overridden kwargs + if llm_router is not None: + return getattr(llm_router, f"{route_type}")(**data) + else: + return getattr(litellm, f"{route_type}")(**data) case _Dispatch.EVAL: - _merge_eval_model_credentials(data, llm_router) + # If a model is provided, get its credentials from the router + model: Final = data.get("model") + if model and llm_router: + try: + # Try to get deployment credentials for this model + deployment_creds = llm_router.get_deployment_credentials(model_id=model) + if not deployment_creds: + # Try by model group name + deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) + if ( + deployment + and deployment.litellm_params + and not llm_router.is_deployment_blocked(deployment) + ): + deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) + + # If we found credentials, merge them into data (but don't override user-provided values) + if deployment_creds: + data.update(deployment_creds) + except Exception: + # If we can't get deployment creds, continue without them + pass + return getattr(litellm, f"{route_type}")(**data) case _TeamModel(name=team_model_name): data["model"] = team_model_name @@ -960,55 +1010,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict) if result is not None: return result + case _Dispatch.BATCH_COMPLETION | _Dispatch.MODEL_NOT_FOUND: + pass _raise_model_not_found(data=data, route_type=route_type) - - -def _merge_router_settings_override( - data: dict, # mutable-ok: merges into the request body route_request dispatches -) -> None: - # Apply per-request router settings overrides from key/team config - # Instead of creating a new Router (expensive), merge settings into kwargs - # The Router already supports per-request overrides for these settings - override_settings: Final = data.pop("router_settings_override") - - # Settings that the Router accepts as per-request kwargs - # These override the global router settings for this specific request - per_request_settings: Final = [ - "fallbacks", - "context_window_fallbacks", - "content_policy_fallbacks", - "num_retries", - "timeout", - "model_group_retry_policy", - "routing_strategy", - "enable_tag_filtering", - ] - - for key in per_request_settings: - if override_settings.get(key) is not None and key not in data: - data[key] = override_settings[key] - - -def _merge_eval_model_credentials( - data: dict, # mutable-ok: merges into the request body route_request dispatches - llm_router: LitellmRouter | None, -) -> None: - # If a model is provided, get its credentials from the router - model: Final = data.get("model") - if model and llm_router: - try: - # Try to get deployment credentials for this model - deployment_creds = llm_router.get_deployment_credentials(model_id=model) - if not deployment_creds: - # Try by model group name - deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model) - if deployment and deployment.litellm_params and not llm_router.is_deployment_blocked(deployment): - deployment_creds = deployment.litellm_params.model_dump(exclude_none=True) - - # If we found credentials, merge them into data (but don't override user-provided values) - if deployment_creds: - data.update(deployment_creds) - except Exception: - # If we can't get deployment creds, continue without them - pass From 4c5f2019faf4bb510adf46beef2ed2a6dcb361dc Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:56:02 -0700 Subject: [PATCH 15/24] chore(proxy): name the untyped router return on the read-through lambda --- litellm/proxy/route_llm_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 7bfb8077990..9a98c88eb62 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -613,7 +613,7 @@ async def route_request( return await _retry_once_after_model_read_through( data, llm_router, - lambda router: _route_request_single_attempt( + lambda router: _route_request_single_attempt( # pyright: ignore[reportUnknownLambdaType] # the router methods it returns are untyped data=data, llm_router=router, user_model=user_model, From e9101b9cd7e82d9a268bf1b067c6cf1c9b3ea399 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:28:09 -0700 Subject: [PATCH 16/24] test(proxy): pin the read-through router, the team model rename and a null user_config route_request retries through the router the registry read-through built, and a request with no model on a team key is sent under the team deployment's public name. raise_if_model_not_routable treats user_config: null as absent, as dispatch does. --- tests/unit/proxy/test_route_llm_request.py | 74 +++++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index aaedca5dc98..90457f8d7a5 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1726,13 +1726,17 @@ def _router_with_gpt4o_mini(): ) +@pytest.mark.parametrize( + "extra_body", + [pytest.param({}, id="plain"), pytest.param({"user_config": None}, id="null-user-config")], +) @pytest.mark.asyncio -async def test_raise_if_model_not_routable_raises_for_unknown_model(): +async def test_raise_if_model_not_routable_raises_for_unknown_model(extra_body): from litellm.proxy.route_llm_request import raise_if_model_not_routable with pytest.raises(ProxyModelNotFoundError) as exc_info: await raise_if_model_not_routable( - data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]}, + data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], **extra_body}, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion", @@ -2021,3 +2025,69 @@ async def test_null_user_config_is_treated_as_absent() -> None: ) assert response.choices[0].message.content == "from the main router" + + +@pytest.mark.asyncio +async def test_route_request_without_a_model_sends_to_the_team_deployment_by_its_public_name(): + import litellm + + router = litellm.Router( + model_list=[ + { + "model_name": "internal-team-gpt", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "from-team"}, + "model_info": {"id": "team-gpt-1", "team_id": "team-a", "team_public_model_name": "team-gpt"}, + } + ] + ) + data = { + "model": None, + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"user_api_key_team_id": "team-a"}, + } + + llm_call = await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion") + response = await llm_call + + assert response.choices[0].message.content == "from-team" + assert data["model"] == "team-gpt" + + +@pytest.mark.asyncio +async def test_route_request_read_through_sends_through_the_router_it_built(monkeypatch): + """With no router loaded yet, a model the read-through finds is sent through the router the read-through built.""" + import litellm + from litellm.proxy import proxy_server + + model_name = "e2e-first-db-model" + built_router = litellm.Router( + model_list=[ + { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-4o", + "api_key": "fake", + "mock_response": "hello-from-new-router", + }, + } + ] + ) + fake_prisma, _ = _fake_prisma_client_with_models([_db_model_row(model_name, "unused")]) + monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma) + monkeypatch.setattr(proxy_server, "store_model_in_db", True) + monkeypatch.setattr(proxy_server, "llm_router", None) + + async def build_router_from_db(prisma_client, proxy_logging_obj): + monkeypatch.setattr(proxy_server, "llm_router", built_router) + + monkeypatch.setattr(proxy_server.proxy_config, "add_deployment", build_router_from_db) + + llm_call = await route_request( + data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]}, + llm_router=None, + user_model=None, + route_type="acompletion", + ) + response = await llm_call + + assert response.choices[0].message.content == "hello-from-new-router" From 445c1ae233cf0d080a17d6369e1e179847cb5f3c Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:28:09 -0700 Subject: [PATCH 17/24] fix(proxy): close redaction gaps on requests rejected before their guardrails - Judge whether a pre-call guardrail applies against the request body too, so a guardrail scoped to streaming or non-streaming requests is honored - Redact tool definitions, which guardrails such as DeepKeep rewrite - A rate-limit fallback attempt rejected before its guardrails keeps its redacted request instead of restoring the primary's - A caller that skips guardrails still runs content enforcers for a model a hook renames - Drop the guard for guardrails_only with skip_content_enforcers, which no caller passes --- litellm/proxy/common_request_processing.py | 20 ++- litellm/proxy/utils.py | 10 +- .../proxy/test_common_request_processing.py | 135 +++++++++++++++++- .../utils/proxy_logging/test_pre_call_hook.py | 12 -- 4 files changed, 153 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index ba428944136..e42676ca88a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1816,6 +1816,7 @@ class ProxyBaseLLMRequestProcessing: self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place self.served_model = served_model self._tags_before_guardrails: frozenset[str] | None = None + self._rejected_before_guardrails = False @property def litellm_call_id(self) -> str | None: @@ -2418,7 +2419,7 @@ class ProxyBaseLLMRequestProcessing: data=self.data, call_type=call_type, skip_guardrails=True, - skip_content_enforcers=True, + skip_content_enforcers=not skip_guardrails, ) refresh_proxy_server_request_body_snapshot(self.data) await raise_if_model_not_routable( @@ -2444,16 +2445,17 @@ class ProxyBaseLLMRequestProcessing: Covers every request field a pre-call guardrail can rewrite on the routes that reach this check: chat and Anthropic messages and system, completion prompt, Responses input and - instructions, embedding and speech input, and Gemini contents and system instruction. The - request body, its proxy_server_request snapshot and message logging all carry the redaction - marker, so failure callbacks, post-call failure hooks and spend rows never see content a - masking guardrail would have rewritten. + instructions, embedding and speech input, tool definitions, and Gemini contents and system + instruction. The request body, its proxy_server_request snapshot and message logging all + carry the redaction marker, so failure callbacks, post-call failure hooks and spend rows + never see content a masking guardrail would have rewritten. """ request_metadata: Final = dict[str, object]() for bucket in _request_metadata_buckets(self.data): request_metadata.update(bucket) - if not proxy_logging_obj.has_pre_call_guardrails(request_metadata): + if not proxy_logging_obj.has_pre_call_guardrails(request_metadata, request_body=self.data): return + self._rejected_before_guardrails = True if "messages" in self.data: self.data["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}] if "contents" in self.data: @@ -2461,6 +2463,8 @@ class ProxyBaseLLMRequestProcessing: for content_key in ("prompt", "input"): if content_key in self.data: self.data[content_key] = "" + if "tools" in self.data: + self.data["tools"] = [] for content_key in ("system", "instructions"): if content_key in self.data: self.data[content_key] = REDACTED_BY_LITELLM @@ -2574,6 +2578,7 @@ class ProxyBaseLLMRequestProcessing: ): continue self.data = independent_snapshot(pristine) + self._rejected_before_guardrails = False self.data["model"] = fallback_model try: return await self.common_processing_pre_call_logic( @@ -2598,7 +2603,8 @@ class ProxyBaseLLMRequestProcessing: raise continue except BaseException: - self.data = rate_limited_data + if not self._rejected_before_guardrails: + self.data = rate_limited_data raise self.data = rate_limited_data diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3c952b16463..65d50c2ef8a 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2397,13 +2397,17 @@ class ProxyLogging: return data - def has_pre_call_guardrails(self, request_metadata: Mapping[str, object]) -> bool: + def has_pre_call_guardrails( + self, request_metadata: Mapping[str, object], request_body: Mapping[str, object] | None = None + ) -> bool: """ Whether anything configured would inspect the content of a request carrying this metadata. Evaluated with the same predicate the pre-call loop uses, so a proxy configured only with post-call guardrails answers False. Callers that must pay a real cost to build the hook's input, such as streaming a batch input file off disk, use this to skip that work. + ``request_body`` supplies the request's top-level fields, such as ``stream``, that a + guardrail scoped to streaming or non-streaming requests reads. A content-enforcing ``CustomLogger`` counts too. It is not a guardrail and has no event hook to consult, but it judges the payload the same way, so a proxy configured only with @@ -2414,7 +2418,7 @@ class ProxyLogging: caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} + probe: Final = {**(request_body or {}), "metadata": dict(request_metadata)} return any( isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=GuardrailEventHooks.pre_call) @@ -2477,8 +2481,6 @@ class ProxyLogging: if guardrails_only and skip_guardrails: raise ValueError("guardrails_only and skip_guardrails are mutually exclusive") - if guardrails_only and skip_content_enforcers: - raise ValueError("guardrails_only and skip_content_enforcers are mutually exclusive") if not guardrails_only: self._init_response_taking_too_long_task(data=data) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 538db9c2cab..6c54a6be2a1 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10943,8 +10943,27 @@ class _EmailMaskingHook(CustomLogger): class _RejectingHook(CustomLogger): + def __init__(self, model: str | None = None) -> None: + super().__init__() + self.model = model + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): - raise HTTPException(status_code=403, detail="rejected by hook") + if self.model in (None, data.get("model")): + raise HTTPException(status_code=403, detail="rejected by hook") + return data + + +class _RateLimitedModelHook(CustomLogger): + def __init__(self, model: str) -> None: + super().__init__() + self.model = model + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError + + if data.get("model") == self.model: + raise ProxyRateLimitError(detail=f"rpm limit reached for {self.model}") + return data class TestPreCallHooksForUnroutableModel: @@ -11165,6 +11184,11 @@ class TestPreCallHooksForUnroutableModel: {"prompt": "", "input": "", "max_tokens": 5}, id="prompt-and-input", ), + pytest.param( + {"tools": [{"type": "function", "function": {"name": "mail", "description": "jane@example.com"}}]}, + {"tools": []}, + id="tool-definitions", + ), ], ) async def test_rejected_request_redacts_every_guardrail_scanned_field(self, monkeypatch, extra_body, expected_body): @@ -11253,6 +11277,115 @@ class TestPreCallHooksForUnroutableModel: assert snapshot["litellm_call_id"] == processor.data["litellm_call_id"] assert snapshot["metadata"] is processor.data["metadata"] + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("stream_scope", "stream", "expected_content"), + [ + pytest.param("streaming", True, "redacted-by-litellm", id="streaming-guardrail-streaming-request"), + pytest.param("streaming", False, "email jane@example.com", id="streaming-guardrail-plain-request"), + pytest.param("non_streaming", True, "email jane@example.com", id="plain-guardrail-streaming-request"), + ], + ) + async def test_rejected_request_is_redacted_when_a_guardrail_scoped_to_its_stream_mode_applies( + self, monkeypatch, stream_scope, stream, expected_content + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _CountingGuardrail() + guardrail.apply_stream_scope(stream_scope) + monkeypatch.setattr(litellm, "callbacks", [guardrail]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run( + monkeypatch, + model="does-not-exist", + llm_router=self._router(), + processor=processor, + extra_body={"stream": stream}, + ) + + assert processor.data["messages"] == [{"role": "user", "content": expected_content}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + + @pytest.mark.asyncio + async def test_caller_that_skips_guardrails_still_runs_content_enforcers_for_a_renamed_model(self, monkeypatch): + enforcer: Final = _CountingContentEnforcer() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), enforcer]) + + data: Final = await self._run( + monkeypatch, model="virtual-model", llm_router=self._router(), skip_guardrails=True + ) + + assert data["model"] == "gpt-4o-mini" + assert enforcer.calls == 1 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("primary", "fallback", "rejecting_hook", "expected_status"), + [ + pytest.param("gpt-4o-mini", "does-not-exist", "unused-model", 400, id="fallback-model-unknown"), + pytest.param("does-not-exist", "gpt-4o-mini", "gpt-4o-mini", 403, id="primary-model-unknown"), + ], + ) + async def test_rate_limit_fallback_logs_the_attempt_rejected_before_its_guardrails( + self, monkeypatch, primary, fallback, rejecting_hook, expected_status + ): + from litellm.proxy import proxy_server + from litellm.proxy.common_utils import registry_read_through + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr( + registry_read_through, + "model_registry_read_through", + registry_read_through.RegistryReadThrough( + resync=registry_read_through._resync_model_deployments, + is_loaded=registry_read_through._model_is_loaded, + ), + ) + monkeypatch.setattr( + litellm, + "callbacks", + [_RateLimitedModelHook(model=primary), _RejectingHook(model=rejecting_hook), _CountingGuardrail()], + ) + router: Final = Router( + model_list=[ + {"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}} + ], + fallbacks=[{primary: [fallback]}], + ) + processor: Final = ProxyBaseLLMRequestProcessing( + data={"model": primary, "messages": [{"role": "user", "content": "email jane@example.com"}]} + ) + + request: Final = Request( + {"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""} + ) + + with pytest.raises((ProxyModelNotFoundError, HTTPException)) as exc_info: + await processor._pre_call_with_fallbacks( + request=request, + general_settings={}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + user_api_key_dict=ProxyUserAPIKeyAuth(), + version=None, + proxy_config=proxy_server.ProxyConfig(), + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + model=None, + route_type="acompletion", + llm_router=router, + ) + + assert exc_info.value.status_code == expected_status + assert processor.data["model"] == "does-not-exist" + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] + @pytest.mark.asyncio async def test_no_router_skips_the_routability_check(self, monkeypatch): guardrail: Final = _CountingGuardrail() diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 7174ebdbc47..0b1eaa4c868 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -1013,15 +1013,3 @@ async def test_skip_content_enforcers_skips_enforcers_but_not_other_hooks(proxy_ assert enforcer.calls == 0 assert accountant.calls == 1 - - -@pytest.mark.asyncio -async def test_skip_content_enforcers_and_guardrails_only_are_mutually_exclusive(proxy_logging): - with pytest.raises(ValueError, match="mutually exclusive"): - await proxy_logging.pre_call_hook( - user_api_key_dict=MagicMock(), - data={"model": "m"}, - call_type="acompletion", - guardrails_only=True, - skip_content_enforcers=True, - ) From a545bf21b6b4deb6c13b66303fd587833213378b Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:52:22 -0700 Subject: [PATCH 18/24] refactor(proxy): skip the pre-call routability check for callers that skip guardrails A caller that skips guardrails has no guardrail to hold back, so its request takes the single pre-call pass and an unknown model gets the routing error from route_request, as on main. Tests inject pass_through_all_models through the Router instead of patching the shared settings. --- litellm/proxy/common_request_processing.py | 32 ++++++++++--------- .../proxy/test_common_request_processing.py | 20 ++++++------ tests/unit/proxy/test_route_llm_request.py | 13 ++++---- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index e42676ca88a..bdae2f191b6 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2336,11 +2336,15 @@ class ProxyBaseLLMRequestProcessing: if self._tags_before_guardrails is None: self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data)) - model_routable: Final = llm_router is None or can_route_with_loaded_models( - data=self.data, - llm_router=llm_router, - user_model=user_model, - route_type=route_type, + model_routable: Final = ( + skip_guardrails + or llm_router is None + or can_route_with_loaded_models( + data=self.data, + llm_router=llm_router, + user_model=user_model, + route_type=route_type, + ) ) prefetch_model = self.data.get("model") if llm_router is not None and isinstance(prefetch_model, str): @@ -2419,7 +2423,7 @@ class ProxyBaseLLMRequestProcessing: data=self.data, call_type=call_type, skip_guardrails=True, - skip_content_enforcers=not skip_guardrails, + skip_content_enforcers=True, ) refresh_proxy_server_request_body_snapshot(self.data) await raise_if_model_not_routable( @@ -2429,16 +2433,14 @@ class ProxyBaseLLMRequestProcessing: route_type=call_type, ) except Exception: - if not skip_guardrails: - self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) + self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) raise - if not skip_guardrails: - self.data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=self.data, - call_type=call_type, - guardrails_only=True, - ) + self.data = await proxy_logging_obj.pre_call_hook( + user_api_key_dict=user_api_key_dict, + data=self.data, + call_type=call_type, + guardrails_only=True, + ) def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None: """Redact the content of a request rejected before the pre-call guardrails that apply to it ran. diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 6c54a6be2a1..29d44a71c87 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -11100,16 +11100,15 @@ class TestPreCallHooksForUnroutableModel: @pytest.mark.asyncio @pytest.mark.parametrize( - ("callbacks", "skip_guardrails", "expected"), + ("callbacks", "expected"), [ - pytest.param(lambda: [_CountingGuardrail()], False, True, id="guardrail-applies"), - pytest.param(lambda: [_CountingContentEnforcer()], False, True, id="content-enforcer"), - pytest.param(lambda: [_ModelRewritingHook()], False, None, id="no-guardrail"), - pytest.param(lambda: [_CountingGuardrail()], True, None, id="guardrails-skipped-by-caller"), + pytest.param(lambda: [_CountingGuardrail()], True, id="guardrail-applies"), + pytest.param(lambda: [_CountingContentEnforcer()], True, id="content-enforcer"), + pytest.param(lambda: [_ModelRewritingHook()], None, id="no-guardrail"), ], ) async def test_rejected_request_is_redacted_only_when_guardrails_were_bypassed( - self, monkeypatch, callbacks, skip_guardrails, expected + self, monkeypatch, callbacks, expected ): from litellm.proxy.route_llm_request import ProxyModelNotFoundError @@ -11122,7 +11121,6 @@ class TestPreCallHooksForUnroutableModel: model="does-not-exist", llm_router=self._router(), processor=processor, - skip_guardrails=skip_guardrails, ) logging_obj: Final = processor.data["litellm_logging_obj"] @@ -11309,9 +11307,12 @@ class TestPreCallHooksForUnroutableModel: assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"] @pytest.mark.asyncio - async def test_caller_that_skips_guardrails_still_runs_content_enforcers_for_a_renamed_model(self, monkeypatch): + async def test_caller_that_skips_guardrails_runs_content_enforcers_and_no_guardrail_for_a_renamed_model( + self, monkeypatch + ): enforcer: Final = _CountingContentEnforcer() - monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), enforcer]) + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), enforcer, guardrail]) data: Final = await self._run( monkeypatch, model="virtual-model", llm_router=self._router(), skip_guardrails=True @@ -11319,6 +11320,7 @@ class TestPreCallHooksForUnroutableModel: assert data["model"] == "gpt-4o-mini" assert enforcer.calls == 1 + assert guardrail.models_seen == [] @pytest.mark.asyncio @pytest.mark.parametrize( diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 90457f8d7a5..06b8cd56e1a 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -13,6 +13,7 @@ from litellm.proxy.route_llm_request import ( ProxyModelNotFoundError, route_request, ) +from litellm.types.router import RouterGeneralSettings @pytest.mark.parametrize( @@ -1713,7 +1714,7 @@ def test_proxy_model_not_found_error_keeps_the_raw_model_only_in_the_client_resp assert error.spend_log_error_message.startswith("/chat/completions: Invalid model name passed in") -def _router_with_gpt4o_mini(): +def _router_with_gpt4o_mini(router_general_settings: RouterGeneralSettings | None = None): import litellm return litellm.Router( @@ -1722,7 +1723,8 @@ def _router_with_gpt4o_mini(): "model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}, } - ] + ], + router_general_settings=router_general_settings, ) @@ -1812,7 +1814,7 @@ async def test_raise_if_model_not_routable_reports_body_errors_before_the_model( ], ) @pytest.mark.asyncio -async def test_raise_if_model_not_routable_allows_routable_requests(monkeypatch, data, route_type, router_mutator): +async def test_raise_if_model_not_routable_allows_routable_requests(data, route_type, router_mutator): from litellm.proxy.route_llm_request import raise_if_model_not_routable if router_mutator == "wildcard": @@ -1826,11 +1828,10 @@ async def test_raise_if_model_not_routable_allows_routable_requests(monkeypatch, } ] ) + elif router_mutator == "pass_through_all_models": + router = _router_with_gpt4o_mini(router_general_settings=RouterGeneralSettings(pass_through_all_models=True)) else: router = _router_with_gpt4o_mini() - if router_mutator == "pass_through_all_models": - # router_general_settings is shared across Router instances; monkeypatch restores it - monkeypatch.setattr(router.router_general_settings, "pass_through_all_models", True) assert ( await raise_if_model_not_routable( From 07369a378f73b360e1545b27108628c735a344f5 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:24:22 -0700 Subject: [PATCH 19/24] fix(proxy): check an unroutable model where the first guardrail would run Replace the two-pass pre-call walk with one walk that awaits the routing check right before the first guardrail pipeline, guardrail or content enforcer that applies. Hooks registered after a guardrail keep their order, so they never see the request content before the guardrails ran. --- litellm/proxy/common_request_processing.py | 42 +++--- litellm/proxy/utils.py | 33 ++++- .../proxy/test_common_request_processing.py | 51 ++++++- .../utils/proxy_logging/test_pre_call_hook.py | 133 ++++++++++++++++-- 4 files changed, 220 insertions(+), 39 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 33db1d0479a..ba631689662 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2400,12 +2400,12 @@ class ProxyBaseLLMRequestProcessing: llm_router: Router | None, user_model: str | None, ) -> None: - """Run the pre-call hooks on self.data, with guardrails last when the requested model cannot route yet. + """Run the pre-call hooks on self.data, checking a model that cannot route yet before the first guardrail. - For such a model, hooks other than guardrails and content enforcers run first and may - rewrite the model. A model that still cannot route raises the routing error before any - guardrail runs, otherwise the guardrails run on the rewritten request. A request those - hooks or the routing check reject is redacted. + For such a model, the hooks that come before the first guardrail or content enforcer run + first and may rewrite the model. A model that still cannot route raises the routing error + there, before any guardrail runs, otherwise the remaining hooks and guardrails run in their + usual order. A request rejected before the check passed is redacted. """ if model_routable: self.data = await proxy_logging_obj.pre_call_hook( @@ -2415,30 +2415,30 @@ class ProxyBaseLLMRequestProcessing: skip_guardrails=skip_guardrails, ) return + passed_routing_check: Final = list[bool]() + + async def raise_unless_routable(request: dict[str, object]) -> None: # mutable-ok: request payload + self.data = request + refresh_proxy_server_request_body_snapshot(request) + await raise_if_model_not_routable( + data=request, + llm_router=llm_router, + user_model=user_model, + route_type=call_type, + ) + passed_routing_check.append(True) + try: self.data = await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, data=self.data, call_type=call_type, - skip_guardrails=True, - skip_content_enforcers=True, - ) - refresh_proxy_server_request_body_snapshot(self.data) - await raise_if_model_not_routable( - data=self.data, - llm_router=llm_router, - user_model=user_model, - route_type=call_type, + before_guardrails=raise_unless_routable, ) except Exception: - self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) + if not passed_routing_check: + self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj) raise - self.data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, - data=self.data, - call_type=call_type, - guardrails_only=True, - ) def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None: """Redact the content of a request rejected before the pre-call guardrails that apply to it ran. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index d4c15f33556..3476ee8c846 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2442,7 +2442,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, - skip_content_enforcers: bool = False, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> None: pass @@ -2455,7 +2455,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, - skip_content_enforcers: bool = False, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> dict: pass @@ -2467,7 +2467,7 @@ class ProxyLogging: guardrails_only: bool = False, skip_guardrails: bool = False, endpoint_type: EndpointType = EndpointType.GENERIC, - skip_content_enforcers: bool = False, + before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload ) -> dict | None: """ Allows users to modify/reject the incoming request to the proxy, without having to deal with parsing Request body. @@ -2481,8 +2481,9 @@ class ProxyLogging: limiting, budget accounting, prompt templates and hanging-request alerting are skipped. Use it to scan a payload that is not itself a request, such as one record of a batch file. - ``skip_content_enforcers`` also skips the non-guardrail callbacks that set - ``enforces_request_content``, the ones ``guardrails_only`` keeps. + ``before_guardrails`` is awaited once with the request, right before the first guardrail + pipeline, guardrail or content enforcer that runs on it, or after the other callbacks when + none does. An exception it raises ends the hook. """ verbose_proxy_logger.debug("Inside Proxy Logging Pre-call hook!") @@ -2533,9 +2534,16 @@ class ProxyLogging: raw_request_snapshot: Final[dict | None] = ( # mutable-ok: same request-payload shape as data independent_snapshot(data) if needs_raw_request_snapshot else None ) + pending_before_guardrails: Final = [before_guardrails] if before_guardrails is not None else [] + + async def reach_guardrails(request: dict[str, object]) -> None: # mutable-ok: request payload + while pending_before_guardrails: + await pending_before_guardrails.pop()(request) try: if not skip_guardrails: + if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(data)): + await reach_guardrails(data) data, _ = await self._maybe_execute_pipelines( # rebind-ok: pipeline edits feed the callback loop below data=data, user_api_key_dict=user_api_key_dict, @@ -2561,6 +2569,7 @@ class ProxyLogging: and (guardrails_only or not caps.has_pre_call_override) ): if data is not None: + await reach_guardrails(data) self._process_guardrail_metadata(data) return data @@ -2591,6 +2600,13 @@ class ProxyLogging: if getattr(_callback, "run_in_parallel", False): continue + if ( + pending_before_guardrails + and _callback.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call) + is True + ): + await reach_guardrails(data) + data = await self._run_sequential_guardrail_callback( callback=_callback, data=data, @@ -2603,13 +2619,15 @@ class ProxyLogging: _callback is not None and isinstance(_callback, CustomLogger) and (not guardrails_only or _callback.enforces_request_content) - and not (skip_content_enforcers and _callback.enforces_request_content) and "async_pre_call_hook" in vars(_callback.__class__) and _callback.__class__.async_pre_call_hook != CustomLogger.async_pre_call_hook ): if call_type in MCP_GUARDRAIL_CALL_TYPES and user_api_key_dict is None: continue + if _callback.enforces_request_content and data is not None: + await reach_guardrails(data) + response: Exception | str | Mapping[str, object] | None = ( await _callback.async_pre_call_hook( user_api_key_dict=user_api_key_dict, @@ -2653,6 +2671,9 @@ class ProxyLogging: end_time=end_time, ) + if data is not None: + await reach_guardrails(data) + if deferred_route_exc is not None and data is not None: data = await self._handle_sensitive_data_route_exception(deferred_route_exc, data, user_api_key_dict) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 22d0aa8343f..c30c5569426 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10984,6 +10984,28 @@ class _EmailMaskingHook(CustomLogger): return data +class _EmailMaskingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__( + guardrail_name="email-masking-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True + ) + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + for message in data["messages"]: + message["content"] = message["content"].replace("jane@example.com", "[EMAIL]") + return data + + +class _ContentRecordingHook(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.contents_seen: list[str] = [] + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.contents_seen.extend(message["content"] for message in data["messages"]) + return data + + class _RejectingHook(CustomLogger): def __init__(self, model: str | None = None) -> None: super().__init__() @@ -11069,7 +11091,7 @@ class TestPreCallHooksForUnroutableModel: ) @pytest.mark.asyncio - async def test_unknown_model_runs_other_hooks_but_no_guardrail_or_content_enforcer(self, monkeypatch): + async def test_unknown_model_runs_only_the_hooks_before_the_first_guardrail(self, monkeypatch): from litellm.proxy.route_llm_request import ProxyModelNotFoundError guardrail: Final = _CountingGuardrail() @@ -11099,6 +11121,31 @@ class TestPreCallHooksForUnroutableModel: assert guardrail.models_seen == ["gpt-4o-mini"] assert enforcer.calls == 1 + @pytest.mark.asyncio + async def test_unknown_model_never_reaches_a_hook_registered_after_a_guardrail(self, monkeypatch): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + guardrail: Final = _EmailMaskingGuardrail() + recorder: Final = _ContentRecordingHook() + rejecting_hook: Final = _RejectingHook() + monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder, rejecting_hook]) + + with pytest.raises(ProxyModelNotFoundError) as exc_info: + await self._run(monkeypatch, model="does-not-exist", llm_router=self._router()) + + assert exc_info.value.status_code == 400 + assert recorder.contents_seen == [] + + @pytest.mark.asyncio + async def test_hook_registered_after_a_guardrail_sees_the_guardrail_output_for_a_renamed_model(self, monkeypatch): + recorder: Final = _ContentRecordingHook() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _EmailMaskingGuardrail(), recorder]) + + data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router()) + + assert data["model"] == "gpt-4o-mini" + assert recorder.contents_seen == ["email [EMAIL]"] + @pytest.mark.asyncio async def test_routable_model_runs_every_hook_once(self, monkeypatch): guardrail: Final = _CountingGuardrail() @@ -11177,7 +11224,7 @@ class TestPreCallHooksForUnroutableModel: @pytest.mark.asyncio async def test_request_a_hook_rejects_before_guardrails_is_redacted(self, monkeypatch): guardrail: Final = _CountingGuardrail() - monkeypatch.setattr(litellm, "callbacks", [guardrail, _RejectingHook()]) + monkeypatch.setattr(litellm, "callbacks", [_RejectingHook(), guardrail]) processor: Final = ProxyBaseLLMRequestProcessing(data={}) with pytest.raises(HTTPException) as exc_info: diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 0b1eaa4c868..4a8492f834f 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -997,19 +997,132 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( ) +class _NamedHook(CustomLogger): + def __init__(self, name: str, seen: list[str]) -> None: + super().__init__() + self.name = name + self.seen = seen + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.name) + return data + + +class _NamedEnforcer(_NamedHook): + enforces_request_content = True + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.name) + return data + + +class _NamedGuardrail(CustomGuardrail): + def __init__(self, name: str, seen: list[str], default_on: bool = True, run_in_parallel: bool = False) -> None: + super().__init__(guardrail_name=name, event_hook=GuardrailEventHooks.pre_call, default_on=default_on) + self.seen = seen + self.run_in_parallel = run_in_parallel + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.seen.append(self.guardrail_name) + return data + + +def _callbacks(spec: str, seen: list[str]) -> list[CustomLogger]: + def build(name: str) -> CustomLogger: + if name.startswith("enforcer"): + return _NamedEnforcer(name, seen) + if name.startswith("off-guardrail"): + return _NamedGuardrail(name, seen, default_on=False) + if name.startswith("parallel-guardrail"): + return _NamedGuardrail(name, seen, run_in_parallel=True) + if name.startswith("guardrail"): + return _NamedGuardrail(name, seen) + return _NamedHook(name, seen) + + return [build(name) for name in spec.split()] + + @pytest.mark.asyncio -async def test_skip_content_enforcers_skips_enforcers_but_not_other_hooks(proxy_logging, monkeypatch): - enforcer = _Enforcer() - accountant = _Accountant() - monkeypatch.setattr(litellm, "callbacks", [enforcer, accountant]) +@pytest.mark.parametrize( + ("callbacks", "expected"), + [ + pytest.param("hook-a guardrail hook-b", ["hook-a", "check", "guardrail", "hook-b"], id="guardrail"), + pytest.param("hook-a enforcer hook-b", ["hook-a", "check", "enforcer", "hook-b"], id="content-enforcer"), + pytest.param( + "off-guardrail hook-a guardrail hook-b", + ["hook-a", "check", "guardrail", "hook-b"], + id="guardrail-that-does-not-apply", + ), + pytest.param( + "hook-a parallel-guardrail hook-b", + ["hook-a", "hook-b", "check", "parallel-guardrail"], + id="parallel-guardrail", + ), + pytest.param("hook-a hook-b", ["hook-a", "hook-b", "check"], id="no-guardrail"), + pytest.param("", ["check"], id="no-callbacks"), + ], +) +async def test_before_guardrails_runs_once_right_before_the_first_guardrail( + proxy_logging, make_user_api_key_auth, monkeypatch, callbacks, expected +): + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks(callbacks, seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + + async def check(request: dict) -> None: + seen.append("check") await proxy_logging.pre_call_hook( - user_api_key_dict=MagicMock(), + user_api_key_dict=make_user_api_key_auth(), data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, call_type="acompletion", - skip_guardrails=True, - skip_content_enforcers=True, + before_guardrails=check, ) - assert enforcer.calls == 0 - assert accountant.calls == 1 + assert seen == expected + + +@pytest.mark.asyncio +async def test_before_guardrails_runs_before_a_guardrail_pipeline(proxy_logging, make_user_api_key_auth, monkeypatch): + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep + + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail-in-pipeline", seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + pipeline: Final = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="guardrail-in-pipeline")]) + + async def check(request: dict) -> None: + seen.append("check") + + await proxy_logging.pre_call_hook( + user_api_key_dict=make_user_api_key_auth(), + data={ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "metadata": {"_guardrail_pipelines": [("policy", pipeline)]}, + }, + call_type="acompletion", + before_guardrails=check, + ) + + assert seen == ["check", "guardrail-in-pipeline", "hook-a"] + + +@pytest.mark.asyncio +async def test_a_failing_before_guardrails_stops_the_walk(proxy_logging, make_user_api_key_auth, monkeypatch): + seen: Final = list[str]() + monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail hook-b", seen)) + proxy_logging.slack_alerting_instance = MagicMock(alerting=None) + + async def reject(request: dict) -> None: + raise HTTPException(status_code=400, detail="model cannot route") + + with pytest.raises(HTTPException, match="model cannot route"): + await proxy_logging.pre_call_hook( + user_api_key_dict=make_user_api_key_auth(), + data={"model": "m", "messages": [{"role": "user", "content": "hi"}]}, + call_type="acompletion", + before_guardrails=reject, + ) + + assert seen == ["hook-a"] From 4a988217da881dd5bcbf206b32584dd9260f7580 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:38:37 -0700 Subject: [PATCH 20/24] test(proxy): pin redaction and request adoption around the routing checkpoint --- .../proxy/test_common_request_processing.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index c30c5569426..e2bb451a172 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -10975,6 +10975,15 @@ class _ModelRewritingHook(CustomLogger): return data +class _ModelReplacingHook(CustomLogger): + def __init__(self, target_model: str) -> None: + super().__init__() + self.target_model = target_model + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + return {**data, "model": self.target_model} + + class _EmailMaskingHook(CustomLogger): async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): data["messages"] = [ @@ -11237,6 +11246,36 @@ class TestPreCallHooksForUnroutableModel: logging_obj: Final = processor.data["litellm_logging_obj"] assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is True + @pytest.mark.asyncio + async def test_request_rejected_after_the_routing_check_passed_is_not_redacted(self, monkeypatch): + guardrail: Final = _CountingGuardrail() + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), guardrail, _RejectingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException) as exc_info: + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + assert exc_info.value.status_code == 403 + assert guardrail.models_seen == ["gpt-4o-mini"] + assert processor.data["messages"] == [{"role": "user", "content": "email jane@example.com"}] + logging_obj: Final = processor.data["litellm_logging_obj"] + assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is not True + + @pytest.mark.asyncio + async def test_request_a_hook_replaced_with_another_unknown_model_is_the_one_rejected_and_redacted( + self, monkeypatch + ): + from litellm.proxy.route_llm_request import ProxyModelNotFoundError + + monkeypatch.setattr(litellm, "callbacks", [_ModelReplacingHook("still-unknown"), _CountingGuardrail()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(ProxyModelNotFoundError): + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + assert processor.data["model"] == "still-unknown" + assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}] + @pytest.mark.asyncio @pytest.mark.parametrize( ("extra_body", "expected_body"), From f0f1834dc2adb21fe79588fa80d3bc6cd58141d8 Mon Sep 17 00:00:00 2001 From: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 19:08:04 -0700 Subject: [PATCH 21/24] test(proxy): pin how the early model check treats key and team router settings --- tests/unit/proxy/test_route_llm_request.py | 50 +++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 7a2c42c9356..c5c5f20523f 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -2185,7 +2185,11 @@ def _router_with_gpt4o_mini(router_general_settings: RouterGeneralSettings | Non @pytest.mark.parametrize( "extra_body", - [pytest.param({}, id="plain"), pytest.param({"user_config": None}, id="null-user-config")], + [ + pytest.param({}, id="plain"), + pytest.param({"user_config": None}, id="null-user-config"), + pytest.param({"router_settings_override": {"num_retries": 3}}, id="key-router-settings"), + ], ) @pytest.mark.asyncio async def test_raise_if_model_not_routable_raises_for_unknown_model(extra_body): @@ -2233,6 +2237,50 @@ async def test_raise_if_model_not_routable_reports_body_errors_before_the_model( ) +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_reports_a_malformed_router_settings_override_like_route_request(): + from litellm.proxy._types import ProxyException + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + data: Final = { + "model": "does-not-exist", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": "not-an-object", + } + + with pytest.raises(ProxyException) as from_check: + await raise_if_model_not_routable( + data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion" + ) + with pytest.raises(ProxyException) as from_route: + await route_request(dict(data), _router_with_gpt4o_mini(), None, "acompletion") + + assert from_check.value.param == from_route.value.param == "router_settings_override" + assert from_check.value.message == from_route.value.message + assert data["router_settings_override"] == "not-an-object" + + +@pytest.mark.asyncio +async def test_raise_if_model_not_routable_leaves_the_router_settings_override_for_route_request(): + from litellm.proxy.route_llm_request import raise_if_model_not_routable + + data: Final = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": {"num_retries": 3}, + } + + await raise_if_model_not_routable( + data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion" + ) + + assert data == { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hi"}], + "router_settings_override": {"num_retries": 3}, + } + + @pytest.mark.parametrize( "data, route_type, router_mutator", [ From a3de32375a9495be911e178651657cbb4abc3a0e Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 26 Oct 2026 18:34:54 -0700 Subject: [PATCH 22/24] fix(proxy): log the request as sent when a guardrail blocks after the model check passed The model check now refreshes the logged request copy only when it fails, so a later block logs the copy taken on arrival, as it did before the check moved --- litellm/proxy/common_request_processing.py | 17 ++++++++++------- .../proxy/test_common_request_processing.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 68d68f7661e..e65cb1961e2 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -2562,13 +2562,16 @@ class ProxyBaseLLMRequestProcessing: async def raise_unless_routable(request: dict[str, object]) -> None: # mutable-ok: request payload self.data = request - refresh_proxy_server_request_body_snapshot(request) - await raise_if_model_not_routable( - data=request, - llm_router=llm_router, - user_model=user_model, - route_type=call_type, - ) + try: + await raise_if_model_not_routable( + data=request, + llm_router=llm_router, + user_model=user_model, + route_type=call_type, + ) + except Exception: + refresh_proxy_server_request_body_snapshot(request) + raise passed_routing_check.append(True) try: diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 5d982bebe8c..eed9f0a5d24 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -12131,6 +12131,22 @@ class TestPreCallHooksForUnroutableModel: logging_obj: Final = processor.data["litellm_logging_obj"] assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is not True + @pytest.mark.asyncio + async def test_request_rejected_after_the_routing_check_passed_logs_the_request_as_sent(self, monkeypatch): + monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _CountingGuardrail(), _RejectingHook()]) + processor: Final = ProxyBaseLLMRequestProcessing(data={}) + + with pytest.raises(HTTPException): + await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor) + + snapshot: Final = processor.data["proxy_server_request"]["body"] + assert processor.data["model"] == "gpt-4o-mini" + assert (snapshot["model"], snapshot["messages"], "litellm_call_id" in snapshot) == ( + "virtual-model", + [{"role": "user", "content": "email jane@example.com"}], + False, + ), snapshot + @pytest.mark.asyncio async def test_request_a_hook_replaced_with_another_unknown_model_is_the_one_rejected_and_redacted( self, monkeypatch From a30e391aa3d0f09388f0081097ad283f26233aa2 Mon Sep 17 00:00:00 2001 From: yucheng Date: Mon, 26 Oct 2026 19:04:49 -0700 Subject: [PATCH 23/24] style(proxy): drop the blank line after the routability check docstring --- litellm/proxy/route_llm_request.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 50e497c0543..a69e51a8e23 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -746,7 +746,6 @@ async def raise_if_model_not_routable( route_type: str, ) -> None: """Raise the error route_request would raise before sending this request, without sending it.""" - _merge_router_settings_override(dict(data)) async def check(router: LitellmRouter | None) -> None: From 91df0dd27a45d930fd6c505d10b6c63c67fff2e7 Mon Sep 17 00:00:00 2001 From: yucheng Date: Tue, 27 Oct 2026 01:49:47 -0700 Subject: [PATCH 24/24] fix(proxy): count only pre-call pipelines as pre-call guardrails A policy with only post-call pipelines no longer makes a rejected request with an unroutable model look like it skipped a pre-call guardrail, so it is logged as the hooks left it instead of redacted --- litellm/proxy/utils.py | 4 ++-- .../proxy/utils/proxy_logging/test_pre_call_hook.py | 10 +++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ff47a7bfd0a..1498ef730e0 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2821,12 +2821,12 @@ class ProxyLogging: hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + probe: Final = {**(request_body or {}), "metadata": dict(request_metadata)} + if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(probe)): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {**(request_body or {}), "metadata": dict(request_metadata)} return any( isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=GuardrailEventHooks.pre_call) diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 4a8492f834f..649598ad8a8 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -266,10 +266,14 @@ def test_has_pre_call_guardrails_is_false_without_callbacks(proxy_logging, monke assert proxy_logging.has_pre_call_guardrails({}) is False -def test_has_pre_call_guardrails_is_true_for_a_configured_pipeline(proxy_logging, monkeypatch): - monkeypatch.setattr(litellm, "callbacks", []) +@pytest.mark.parametrize(("mode", "expected"), [("pre_call", True), ("post_call", False)]) +def test_has_pre_call_guardrails_follows_the_pipeline_mode(proxy_logging, monkeypatch, mode, expected): + from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep - assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": ["p1"]}) is True + monkeypatch.setattr(litellm, "callbacks", []) + pipeline: Final = GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="g")]) + + assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": [("policy", pipeline)]}) is expected @pytest.mark.asyncio