diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9117f7417ac..a8a39b9d7ac 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -2580,26 +2580,6 @@ class ProxyLogging: raise HTTPException(status_code=400, detail={"error": response}) return data - def _should_use_guardrail_load_balancing( - self, - guardrail_name: str, - ) -> bool: - """ - Check if load balancing should be used for this guardrail. - - Returns True if the router has multiple deployments for this guardrail name. - """ - from litellm.proxy.proxy_server import llm_router - - if llm_router is None: - return False - router_settings: Final = llm_router.settings() - if not router_settings.is_set("guardrail_list"): - return False - - matching: Final = [g for g in router_settings.guardrail_list if g.get("guardrail_name") == guardrail_name] - return len(matching) > 1 - async def _execute_guardrail_hook( self, callback: "CustomGuardrail", @@ -2657,50 +2637,6 @@ class ProxyLogging: else: raise ValueError(f"Unknown hook_type: {hook_type}") - async def _execute_guardrail_with_load_balancing( - self, - guardrail_name: str, - hook_type: str, - data: dict, - user_api_key_dict: UserAPIKeyAuth | None, - call_type: CallTypesLiteral, - response: LLMResponseTypes | None = None, - ) -> object: - """ - Execute a guardrail using the router's load balancing. - - Args: - guardrail_name: Name of the guardrail - hook_type: One of "pre_call", "during_call", "post_call" - data: Request data - user_api_key_dict: User API key auth - call_type: Type of call - response: Response object (for post_call hooks) - - Returns: - Result from the guardrail execution - """ - from litellm.proxy.proxy_server import llm_router - - if llm_router is None: - raise ValueError("Router not initialized") - - # Select guardrail using router's load balancing - selected_guardrail: Final = llm_router.settings().guardrail(guardrail_name) - - callback: Final[CustomGuardrail | None] = selected_guardrail.get("callback") - if callback is None: - raise ValueError(f"No callback found for guardrail: {guardrail_name}") - - return await self._execute_guardrail_hook( - callback=callback, - hook_type=hook_type, - data=data, - user_api_key_dict=user_api_key_dict, - call_type=call_type, - response=response, - ) - async def _process_guardrail_callback( self, callback: CustomGuardrail, @@ -4845,7 +4781,19 @@ class ProxyLogging: pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines) selected_iterator_callbacks: Final = _load_balanced_callbacks(tuple(cb for cb, _ in caps.iterator_overrides)) selected_iterator_overrides: Final = tuple( - (cb, "override" if "async_post_call_streaming_iterator_hook" in type(cb).__dict__ else "apply_guardrail") + ( + cb, + next( + kind + for original, kind in caps.iterator_overrides + if original is cb + or ( + isinstance(original, CustomGuardrail) + and isinstance(cb, CustomGuardrail) + and original.guardrail_name == cb.guardrail_name + ) + ), + ) for cb in selected_iterator_callbacks ) running_overrides: Final = tuple( diff --git a/tests/unit/proxy/guardrails/test_guardrail_load_balancing.py b/tests/unit/proxy/guardrails/test_guardrail_load_balancing.py index 88f7ef3b1bb..36dbc9c3f8e 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_load_balancing.py +++ b/tests/unit/proxy/guardrails/test_guardrail_load_balancing.py @@ -48,6 +48,10 @@ class MockGuardrail(CustomGuardrail): yield chunk +class InheritedIteratorGuardrail(MockGuardrail): + pass + + @pytest.mark.asyncio @pytest.mark.parametrize("hook", ("pre_call", "during_call", "post_call", "streaming", "iterator")) @pytest.mark.parametrize("deployment_count", (0, 1, 2)) @@ -56,7 +60,10 @@ async def test_proxy_logging_hooks_load_balance_once_per_request(hook: str, depl MockGuardrail.call_count = 0 guardrail_1 = MockGuardrail(guardrail_name="content-filter", guardrail_id="g1") - guardrail_2 = MockGuardrail(guardrail_name="content-filter", guardrail_id="g2") + second_guardrail_class: Final = ( + InheritedIteratorGuardrail if hook == "iterator" and deployment_count == 2 else MockGuardrail + ) + guardrail_2 = second_guardrail_class(guardrail_name="content-filter", guardrail_id="g2") guardrail_list = [ { diff --git a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py index 5fd40696224..b37b5136b14 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py +++ b/tests/unit/proxy/utils/proxy_logging/test_guardrail_pipeline.py @@ -1,7 +1,6 @@ """Pin ProxyLogging guardrail pipeline helpers. -Covers ``_should_use_guardrail_load_balancing``, ``_execute_guardrail_hook``, -``_execute_guardrail_with_load_balancing``, ``_process_guardrail_callback``, +Covers ``_execute_guardrail_hook``, ``_process_guardrail_callback``, ``_process_prompt_template``, ``_process_guardrail_metadata``, ``_maybe_execute_pipelines``, ``_handle_pipeline_result``, ``_run_guardrail_with_metrics``, ``_emit_guardrail_metrics``. @@ -18,8 +17,6 @@ import logging from typing import Any, Callable, Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch -from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs - import pytest from fastapi import HTTPException @@ -60,39 +57,6 @@ def _clear_caps_cache(): ProxyLogging._callback_capabilities_cache.clear() -# --------------------------------------------------------------------------- -# _should_use_guardrail_load_balancing -# --------------------------------------------------------------------------- - - -def test_should_use_guardrail_load_balancing_truth_table(proxy_logging): - snapshot = {} - router = route_entry_points_to_stubs(MagicMock()) - router.guardrail_list = [{"guardrail_name": "g1"}, {"guardrail_name": "g1"}] - with patch("litellm.proxy.proxy_server.llm_router", router): - snapshot["multiple_deployments"] = proxy_logging._should_use_guardrail_load_balancing("g1") - router.guardrail_list = [{"guardrail_name": "g1"}] - with patch("litellm.proxy.proxy_server.llm_router", router): - snapshot["single_deployment"] = proxy_logging._should_use_guardrail_load_balancing("g1") - with patch("litellm.proxy.proxy_server.llm_router", None): - snapshot["no_router"] = proxy_logging._should_use_guardrail_load_balancing("g1") - router.guardrail_list = [{"guardrail_name": "other"}, {"guardrail_name": "other"}] - with patch("litellm.proxy.proxy_server.llm_router", router): - snapshot["unmatched_name"] = proxy_logging._should_use_guardrail_load_balancing("g1") - assert snapshot == { - "multiple_deployments": True, - "single_deployment": False, - "no_router": False, - "unmatched_name": False, - } - - -def test_should_use_guardrail_load_balancing_error_on_bad_guardrail_list(proxy_logging): - router = route_entry_points_to_stubs(MagicMock()) - router.guardrail_list = "not a list" - with patch("litellm.proxy.proxy_server.llm_router", router): - with pytest.raises((TypeError, AttributeError)): - proxy_logging._should_use_guardrail_load_balancing("g1") # --------------------------------------------------------------------------- @@ -165,53 +129,6 @@ async def test_execute_guardrail_hook_unknown_hook_type_raises(proxy_logging, ma ) -# --------------------------------------------------------------------------- -# _execute_guardrail_with_load_balancing -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_routes_through_router(proxy_logging, make_user_api_key_auth): - cb = _make_guardrail() - router = route_entry_points_to_stubs(MagicMock()) - router.get_available_guardrail = MagicMock(return_value={"callback": cb}) - with patch("litellm.proxy.proxy_server.llm_router", router): - out = await proxy_logging._execute_guardrail_with_load_balancing( - guardrail_name="g", - hook_type="pre_call", - data={"model": "m"}, - user_api_key_dict=make_user_api_key_auth(), - call_type="completion", - ) - assert out == {"a": 1, "b": 2, "c": 3} - - -@pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_router_none_raises(proxy_logging, make_user_api_key_auth): - with patch("litellm.proxy.proxy_server.llm_router", None): - with pytest.raises(ValueError, match="Router not initialized"): - await proxy_logging._execute_guardrail_with_load_balancing( - guardrail_name="g", - hook_type="pre_call", - data={}, - user_api_key_dict=make_user_api_key_auth(), - call_type="completion", - ) - - -@pytest.mark.asyncio -async def test_execute_guardrail_with_load_balancing_no_callback_raises(proxy_logging, make_user_api_key_auth): - router = route_entry_points_to_stubs(MagicMock()) - router.get_available_guardrail = MagicMock(return_value={"callback": None}) - with patch("litellm.proxy.proxy_server.llm_router", router): - with pytest.raises(ValueError, match="No callback found"): - await proxy_logging._execute_guardrail_with_load_balancing( - guardrail_name="g", - hook_type="pre_call", - data={}, - user_api_key_dict=make_user_api_key_auth(), - call_type="completion", - ) # --------------------------------------------------------------------------- @@ -237,7 +154,6 @@ async def test_process_guardrail_callback_skipped_when_should_run_false(proxy_lo async def test_process_guardrail_callback_returns_data_on_success(proxy_logging, make_user_api_key_auth, monkeypatch): cb = _make_guardrail() cb.should_run_guardrail = MagicMock(return_value=True) - proxy_logging._should_use_guardrail_load_balancing = MagicMock(return_value=False) out = await proxy_logging._process_guardrail_callback( callback=cb, data={"model": "m", "messages": [{"role": "user"}], "temperature": 0.1}, @@ -257,7 +173,6 @@ async def test_process_guardrail_callback_enriches_and_reraises_http_exception( detail = {"error": "blocked"} cb.async_pre_call_hook = AsyncMock(side_effect=HTTPException(status_code=400, detail=detail)) cb.event_hook = "pre_call" - proxy_logging._should_use_guardrail_load_balancing = MagicMock(return_value=False) with pytest.raises(HTTPException): await proxy_logging._process_guardrail_callback( @@ -4255,4 +4170,3 @@ async def test_live_pipeline_disconnect_scan_that_cannot_run_records_every_step_ assert seen.get("count") is None assert _post_call_statuses(data) == ["guardrail_failed_to_respond"] assert any("cut short" in message and "RuntimeError" in message for message in _warnings(caplog)) -