fix(guardrails): preserve cached streaming iterator hook kinds

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
mateo 2026-11-05 16:48:27 +00:00
parent e66f38e447
commit f1c79e7e4b
3 changed files with 22 additions and 153 deletions

View file

@ -2580,26 +2580,6 @@ class ProxyLogging:
raise HTTPException(status_code=400, detail={"error": response})
return data
def _should_use_guardrail_load_balancing(
self,
guardrail_name: str,
) -> bool:
"""
Check if load balancing should be used for this guardrail.
Returns True if the router has multiple deployments for this guardrail name.
"""
from litellm.proxy.proxy_server import llm_router
if llm_router is None:
return False
router_settings: Final = llm_router.settings()
if not router_settings.is_set("guardrail_list"):
return False
matching: Final = [g for g in router_settings.guardrail_list if g.get("guardrail_name") == guardrail_name]
return len(matching) > 1
async def _execute_guardrail_hook(
self,
callback: "CustomGuardrail",
@ -2657,50 +2637,6 @@ class ProxyLogging:
else:
raise ValueError(f"Unknown hook_type: {hook_type}")
async def _execute_guardrail_with_load_balancing(
self,
guardrail_name: str,
hook_type: str,
data: dict,
user_api_key_dict: UserAPIKeyAuth | None,
call_type: CallTypesLiteral,
response: LLMResponseTypes | None = None,
) -> object:
"""
Execute a guardrail using the router's load balancing.
Args:
guardrail_name: Name of the guardrail
hook_type: One of "pre_call", "during_call", "post_call"
data: Request data
user_api_key_dict: User API key auth
call_type: Type of call
response: Response object (for post_call hooks)
Returns:
Result from the guardrail execution
"""
from litellm.proxy.proxy_server import llm_router
if llm_router is None:
raise ValueError("Router not initialized")
# Select guardrail using router's load balancing
selected_guardrail: Final = llm_router.settings().guardrail(guardrail_name)
callback: Final[CustomGuardrail | None] = selected_guardrail.get("callback")
if callback is None:
raise ValueError(f"No callback found for guardrail: {guardrail_name}")
return await self._execute_guardrail_hook(
callback=callback,
hook_type=hook_type,
data=data,
user_api_key_dict=user_api_key_dict,
call_type=call_type,
response=response,
)
async def _process_guardrail_callback(
self,
callback: CustomGuardrail,
@ -4845,7 +4781,19 @@ class ProxyLogging:
pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
selected_iterator_callbacks: Final = _load_balanced_callbacks(tuple(cb for cb, _ in caps.iterator_overrides))
selected_iterator_overrides: Final = tuple(
(cb, "override" if "async_post_call_streaming_iterator_hook" in type(cb).__dict__ else "apply_guardrail")
(
cb,
next(
kind
for original, kind in caps.iterator_overrides
if original is cb
or (
isinstance(original, CustomGuardrail)
and isinstance(cb, CustomGuardrail)
and original.guardrail_name == cb.guardrail_name
)
),
)
for cb in selected_iterator_callbacks
)
running_overrides: Final = tuple(

View file

@ -48,6 +48,10 @@ class MockGuardrail(CustomGuardrail):
yield chunk
class InheritedIteratorGuardrail(MockGuardrail):
pass
@pytest.mark.asyncio
@pytest.mark.parametrize("hook", ("pre_call", "during_call", "post_call", "streaming", "iterator"))
@pytest.mark.parametrize("deployment_count", (0, 1, 2))
@ -56,7 +60,10 @@ async def test_proxy_logging_hooks_load_balance_once_per_request(hook: str, depl
MockGuardrail.call_count = 0
guardrail_1 = MockGuardrail(guardrail_name="content-filter", guardrail_id="g1")
guardrail_2 = MockGuardrail(guardrail_name="content-filter", guardrail_id="g2")
second_guardrail_class: Final = (
InheritedIteratorGuardrail if hook == "iterator" and deployment_count == 2 else MockGuardrail
)
guardrail_2 = second_guardrail_class(guardrail_name="content-filter", guardrail_id="g2")
guardrail_list = [
{

View file

@ -1,7 +1,6 @@
"""Pin ProxyLogging guardrail pipeline helpers.
Covers ``_should_use_guardrail_load_balancing``, ``_execute_guardrail_hook``,
``_execute_guardrail_with_load_balancing``, ``_process_guardrail_callback``,
Covers ``_execute_guardrail_hook``, ``_process_guardrail_callback``,
``_process_prompt_template``, ``_process_guardrail_metadata``,
``_maybe_execute_pipelines``, ``_handle_pipeline_result``,
``_run_guardrail_with_metrics``, ``_emit_guardrail_metrics``.
@ -18,8 +17,6 @@ import logging
from typing import Any, Callable, Dict, Final, List
from unittest.mock import AsyncMock, MagicMock, patch
from tests.unit.proxy.router_entry_point_stubs import route_entry_points_to_stubs
import pytest
from fastapi import HTTPException
@ -60,39 +57,6 @@ def _clear_caps_cache():
ProxyLogging._callback_capabilities_cache.clear()
# ---------------------------------------------------------------------------
# _should_use_guardrail_load_balancing
# ---------------------------------------------------------------------------
def test_should_use_guardrail_load_balancing_truth_table(proxy_logging):
snapshot = {}
router = route_entry_points_to_stubs(MagicMock())
router.guardrail_list = [{"guardrail_name": "g1"}, {"guardrail_name": "g1"}]
with patch("litellm.proxy.proxy_server.llm_router", router):
snapshot["multiple_deployments"] = proxy_logging._should_use_guardrail_load_balancing("g1")
router.guardrail_list = [{"guardrail_name": "g1"}]
with patch("litellm.proxy.proxy_server.llm_router", router):
snapshot["single_deployment"] = proxy_logging._should_use_guardrail_load_balancing("g1")
with patch("litellm.proxy.proxy_server.llm_router", None):
snapshot["no_router"] = proxy_logging._should_use_guardrail_load_balancing("g1")
router.guardrail_list = [{"guardrail_name": "other"}, {"guardrail_name": "other"}]
with patch("litellm.proxy.proxy_server.llm_router", router):
snapshot["unmatched_name"] = proxy_logging._should_use_guardrail_load_balancing("g1")
assert snapshot == {
"multiple_deployments": True,
"single_deployment": False,
"no_router": False,
"unmatched_name": False,
}
def test_should_use_guardrail_load_balancing_error_on_bad_guardrail_list(proxy_logging):
router = route_entry_points_to_stubs(MagicMock())
router.guardrail_list = "not a list"
with patch("litellm.proxy.proxy_server.llm_router", router):
with pytest.raises((TypeError, AttributeError)):
proxy_logging._should_use_guardrail_load_balancing("g1")
# ---------------------------------------------------------------------------
@ -165,53 +129,6 @@ async def test_execute_guardrail_hook_unknown_hook_type_raises(proxy_logging, ma
)
# ---------------------------------------------------------------------------
# _execute_guardrail_with_load_balancing
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_execute_guardrail_with_load_balancing_routes_through_router(proxy_logging, make_user_api_key_auth):
cb = _make_guardrail()
router = route_entry_points_to_stubs(MagicMock())
router.get_available_guardrail = MagicMock(return_value={"callback": cb})
with patch("litellm.proxy.proxy_server.llm_router", router):
out = await proxy_logging._execute_guardrail_with_load_balancing(
guardrail_name="g",
hook_type="pre_call",
data={"model": "m"},
user_api_key_dict=make_user_api_key_auth(),
call_type="completion",
)
assert out == {"a": 1, "b": 2, "c": 3}
@pytest.mark.asyncio
async def test_execute_guardrail_with_load_balancing_router_none_raises(proxy_logging, make_user_api_key_auth):
with patch("litellm.proxy.proxy_server.llm_router", None):
with pytest.raises(ValueError, match="Router not initialized"):
await proxy_logging._execute_guardrail_with_load_balancing(
guardrail_name="g",
hook_type="pre_call",
data={},
user_api_key_dict=make_user_api_key_auth(),
call_type="completion",
)
@pytest.mark.asyncio
async def test_execute_guardrail_with_load_balancing_no_callback_raises(proxy_logging, make_user_api_key_auth):
router = route_entry_points_to_stubs(MagicMock())
router.get_available_guardrail = MagicMock(return_value={"callback": None})
with patch("litellm.proxy.proxy_server.llm_router", router):
with pytest.raises(ValueError, match="No callback found"):
await proxy_logging._execute_guardrail_with_load_balancing(
guardrail_name="g",
hook_type="pre_call",
data={},
user_api_key_dict=make_user_api_key_auth(),
call_type="completion",
)
# ---------------------------------------------------------------------------
@ -237,7 +154,6 @@ async def test_process_guardrail_callback_skipped_when_should_run_false(proxy_lo
async def test_process_guardrail_callback_returns_data_on_success(proxy_logging, make_user_api_key_auth, monkeypatch):
cb = _make_guardrail()
cb.should_run_guardrail = MagicMock(return_value=True)
proxy_logging._should_use_guardrail_load_balancing = MagicMock(return_value=False)
out = await proxy_logging._process_guardrail_callback(
callback=cb,
data={"model": "m", "messages": [{"role": "user"}], "temperature": 0.1},
@ -257,7 +173,6 @@ async def test_process_guardrail_callback_enriches_and_reraises_http_exception(
detail = {"error": "blocked"}
cb.async_pre_call_hook = AsyncMock(side_effect=HTTPException(status_code=400, detail=detail))
cb.event_hook = "pre_call"
proxy_logging._should_use_guardrail_load_balancing = MagicMock(return_value=False)
with pytest.raises(HTTPException):
await proxy_logging._process_guardrail_callback(
@ -4255,4 +4170,3 @@ async def test_live_pipeline_disconnect_scan_that_cannot_run_records_every_step_
assert seen.get("count") is None
assert _post_call_statuses(data) == ["guardrail_failed_to_respond"]
assert any("cut short" in message and "RuntimeError" in message for message in _warnings(caplog))