fix(proxy): run enterprise folder guardrails and stop naming skipped guardrails as applied
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

A guardrail from an operator's config-side enterprise/ folder now counts as operator code, so it and every guardrail before it still run for a model routing refuses. The shipped enterprise.enterprise_hooks modules count the same way, which runs the guardrails before them as on main.

A guardrail the walk skipped for a refused model is left out of applied_guardrails, on the normal, reroute and error exits of the walk.
This commit is contained in:
yucheng 2026-11-02 14:26:39 -08:00
parent e5a1eb365f
commit b1ebdab063
3 changed files with 135 additions and 18 deletions

View file

@ -793,9 +793,18 @@ class UnroutableModelProbe:
recognizes and an ``a2a/`` agent name count as routable without asking.
``refusal`` is the routing error a caller skipped a pre-call guardrail for, and the request must end with it.
``skipped_guardrails`` names the guardrails the caller skipped for it.
"""
__slots__ = ("_checked", "_llm_router", "_route_type", "_user_api_key_dict", "_user_model", "refusal")
__slots__ = (
"_checked",
"_llm_router",
"_route_type",
"_user_api_key_dict",
"_user_model",
"refusal",
"skipped_guardrails",
)
def __init__(
self,
@ -810,6 +819,7 @@ class UnroutableModelProbe:
self._user_api_key_dict: Final = user_api_key_dict
self._checked: tuple[object, ProxyModelNotFoundError | None] | None = None
self.refusal: ProxyModelNotFoundError | None = None
self.skipped_guardrails: frozenset[str] = frozenset()
async def refuses(self, data: Mapping[str, object]) -> bool:
model: Final = data.get("model")
@ -820,6 +830,10 @@ class UnroutableModelProbe:
self.refusal = refusal
return refusal is not None
def record_skipped_guardrail(self, guardrail_name: str | None) -> None:
if guardrail_name is not None:
self.skipped_guardrails |= {guardrail_name}
async def _refusal(self, data: Mapping[str, object], model: object) -> ProxyModelNotFoundError | None:
if _is_a2a_agent_model(model) or (
isinstance(model, str)

View file

@ -1892,7 +1892,7 @@ def _overrides_moderation_hook(callback: CustomLogger) -> bool:
return _overrides_hook(callback, "async_moderation_hook")
_LITELLM_PACKAGES: Final = frozenset({"litellm", "litellm_enterprise", "enterprise"})
_LITELLM_PACKAGES: Final = frozenset({"litellm", "litellm_enterprise"})
def _is_custom_pre_call_hook(callback: CustomLogger) -> bool:
@ -1912,12 +1912,15 @@ async def _skipped_for_unroutable_model(
event_type: Final = (
GuardrailEventHooks.pre_mcp_call if call_type in MCP_GUARDRAIL_CALL_TYPES else GuardrailEventHooks.pre_call
)
return (
probe is not None
and caps.resolved_callbacks.index(guardrail) > caps.last_custom_pre_call_hook
and guardrail.should_run_guardrail(data=data, event_type=event_type) # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
and await probe.refuses(data)
)
if (
probe is None
or caps.resolved_callbacks.index(guardrail) <= caps.last_custom_pre_call_hook
or not guardrail.should_run_guardrail(data=data, event_type=event_type) # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
or not await probe.refuses(data)
):
return False
probe.record_skipped_guardrail(guardrail.guardrail_name)
return True
_SYSTEM_INSTRUCTION_KEYS: Final = ("systemInstruction", "system_instruction")
@ -3011,11 +3014,13 @@ class ProxyLogging:
data.pop("prompt_version", None)
data.pop("prompt_environment", None)
def _process_guardrail_metadata(self, data: dict, *, skip_guardrails: bool = False) -> None:
def _process_guardrail_metadata(
self, data: dict, *, skip_guardrails: bool = False, routing_probe: "UnroutableModelProbe | None" = None
) -> None:
"""
Process guardrails from metadata and add to applied_guardrails, leaving out the pre_call
policy pipeline steps that have not run. With ``skip_guardrails`` the pipelines run on
another request, so every step is added.
policy pipeline steps that have not run and the guardrails ``routing_probe`` skipped.
With ``skip_guardrails`` the pipelines run on another request, so every step is added.
"""
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
@ -3044,7 +3049,7 @@ class ProxyLogging:
frozenset[str]()
if skip_guardrails
else _request_steps_that_have_not_run(cast(dict[str, object], data)) # cast-ok: request payload
)
) | (frozenset[str]() if routing_probe is None else routing_probe.skipped_guardrails)
for guardrail_name in guardrails_in_metadata:
if (
isinstance(guardrail_name, str)
@ -3287,7 +3292,7 @@ class ProxyLogging:
With ``routing_probe``, a pre-call guardrail litellm ships is not called for a model ``route_request``
refuses, when only litellm's own guardrails and hooks come after it and no guardrail reroutes on
sensitive data. The hooks after a skipped sequential guardrail see the request with every field it
could have rewritten redacted.
could have rewritten redacted, and a skipped guardrail is left out of ``applied_guardrails``.
"""
verbose_proxy_logger.debug("Inside Proxy Logging Pre-call hook!")
@ -3493,17 +3498,17 @@ class ProxyLogging:
)
if data is not None:
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails)
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails, routing_probe=routing_probe)
return data
except SensitiveDataRouteException as e:
data = await self._handle_sensitive_data_route_exception(e, data, user_api_key_dict)
if data is not None:
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails)
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails, routing_probe=routing_probe)
return data
except Exception:
if data is not None:
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails)
self._process_guardrail_metadata(data, skip_guardrails=skip_guardrails, routing_probe=routing_probe)
raise
async def _run_parallel_pre_call_guardrails(

View file

@ -1,4 +1,5 @@
import asyncio
import contextlib
import json
from datetime import datetime
from typing import Any, Dict, Final, List, Optional, TypeAlias, Union
@ -8,6 +9,7 @@ import pytest
from fastapi import HTTPException, Request
from starlette.datastructures import State
from litellm.exceptions import SensitiveDataRouteException
from litellm.integrations.custom_guardrail import CustomGuardrail, ModifyResponseException
from litellm.integrations.custom_logger import CustomLogger
from litellm.proxy.utils import get_docs_url, get_openapi_url, get_redoc_url
@ -3785,7 +3787,7 @@ def _unroutable_model_probe():
)
async def _walk(monkeypatch, callbacks, model: str, probe, call_type: str = "completion") -> dict:
async def _walk(monkeypatch, callbacks, model: str, probe, call_type: str = "completion", metadata=None) -> dict:
from litellm.caching.caching import DualCache
from litellm.proxy.utils import ProxyLogging
@ -3793,7 +3795,11 @@ async def _walk(monkeypatch, callbacks, model: str, probe, call_type: str = "com
ProxyLogging._callback_capabilities_cache.clear()
return await ProxyLogging(user_api_key_cache=DualCache()).pre_call_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="test_key", user_id="test_user"),
data={"model": model, "messages": [dict(message) for message in _SECRET_PROMPT]},
data={
"model": model,
"messages": [dict(message) for message in _SECRET_PROMPT],
**({} if metadata is None else {"metadata": metadata}),
},
call_type=call_type,
routing_probe=probe,
)
@ -3839,6 +3845,24 @@ async def test_pre_call_hook_runs_an_operator_guardrail_and_every_guardrail_befo
assert (len(before.seen), len(operator.seen), after.seen) == (1, 1, ())
@pytest.mark.parametrize(
("module_name", "runs"),
[
pytest.param("enterprise.op_hooks", True, id="operator-folder-named-enterprise"),
pytest.param("enterprise.enterprise_hooks.banned_keywords", True, id="enterprise-hook"),
pytest.param("litellm_enterprise.enterprise_callbacks.secret_detection", False, id="litellm-enterprise"),
],
)
async def test_pre_call_hook_counts_only_the_litellm_packages_as_shipped(monkeypatch, module_name, runs):
guardrail_class: Final = type("ModuleGuardrail", (_ShippedGuardrail,), {"__module__": module_name})
before: Final = _ShippedGuardrail(name="before")
guardrail: Final = guardrail_class(name="in-module")
await _walk(monkeypatch, [before, guardrail], "no-such-model", _unroutable_model_probe())
assert (len(before.seen), len(guardrail.seen)) == ((1, 1) if runs else (0, 0))
async def test_pre_call_hook_runs_a_custom_code_guardrail_for_a_model_routing_refuses(monkeypatch):
from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import CustomCodeGuardrail
@ -3879,6 +3903,80 @@ async def test_pre_call_hook_hides_the_prompt_from_hooks_after_a_skipped_guardra
assert result["messages"] == redacted
@pytest.mark.parametrize(
("model", "armed", "applied"),
[
pytest.param("no-such-model", True, None, id="refused-model"),
pytest.param("gpt-4o-mini", True, ["opt-in"], id="routable-model"),
pytest.param("no-such-model", False, ["opt-in"], id="no-probe"),
],
)
async def test_pre_call_hook_names_an_opt_in_guardrail_only_when_it_ran(monkeypatch, model, armed, applied):
guardrail: Final = _ShippedGuardrail(name="opt-in", default_on=False)
result: Final = await _walk(
monkeypatch,
[guardrail],
model,
_unroutable_model_probe() if armed else None,
metadata={"guardrails": ["opt-in"]},
)
assert result["metadata"].get("applied_guardrails") == applied
assert len(guardrail.seen) == (0 if applied is None else 1)
@pytest.mark.parametrize("after_in_parallel", [False, True], ids=["after-sequential", "after-parallel"])
async def test_pre_call_hook_names_the_guardrails_before_an_operator_guardrail_and_not_the_one_after(
monkeypatch, after_in_parallel
):
before: Final = _ShippedGuardrail(name="before", default_on=False)
operator: Final = _OperatorGuardrail(name="operator", default_on=False)
after: Final = _ShippedGuardrail(name="after", default_on=False, run_in_parallel=after_in_parallel)
result: Final = await _walk(
monkeypatch,
[before, operator, after],
"no-such-model",
_unroutable_model_probe(),
metadata={"guardrails": ["before", "operator", "after"]},
)
assert after.seen == ()
assert result["metadata"]["applied_guardrails"] == ["before", "operator"]
@pytest.mark.parametrize(
"error",
[
pytest.param(ValueError("operator rule"), id="operator-blocks"),
pytest.param(
SensitiveDataRouteException(route_to_model="gpt-4o-mini", session_id="s1", sticky_session_routing=False),
id="operator-reroutes",
),
],
)
async def test_pre_call_hook_leaves_a_skipped_guardrail_out_when_a_parallel_guardrail_ends_the_walk(monkeypatch, error):
class _EndsTheWalk(_OperatorGuardrail):
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
raise error
after: Final = _ShippedGuardrail(name="after", default_on=False)
metadata: Final[dict[str, object]] = {"guardrails": ["operator", "after"]}
with contextlib.suppress(ValueError):
await _walk(
monkeypatch,
[_EndsTheWalk(name="operator", default_on=False, run_in_parallel=True), after],
"no-such-model",
_unroutable_model_probe(),
metadata=metadata,
)
assert after.seen == ()
assert metadata["applied_guardrails"] == ["operator"]
@pytest.mark.parametrize(
("event_hook", "guardrail_skipped"),
[