This commit is contained in:
yucheng-berri 2026-11-03 02:08:19 +00:00 • committed by GitHub
commit 071c303950
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 1902 additions and 120 deletions

View file

@ -211,46 +211,34 @@ def guardrail_request_data_with_streaming(
}
_GUARDRAIL_ENTRY_WITH_SETTINGS: Final = TypeAdapter(Mapping[str, object])
def merged_guardrail_entries(*entry_lists: Iterable[object]) -> list[object]:
"""Merges guardrail lists so each guardrail is named once: the first ``{name: settings}`` entry naming it wins over
any plain name in either order, else its first plain name stays"""
entries: Final = tuple(chain.from_iterable(entry_lists))
settings_names: Final = tuple(_settings_entry_names(entry) for entry in entries)
named_with_settings: Final = frozenset[str]().union(*settings_names)
return [
entry
for index, entry in enumerate(entries)
if not _superseded_guardrail_entry(entries, settings_names, named_with_settings, index)
]
if all(isinstance(entry, str) for entry in entries):
return list(dict.fromkeys(entries))
named_with_settings: Final = frozenset(
chain.from_iterable(guardrail_entry_names(entry) for entry in entries if isinstance(entry, dict))
)
seen: Final[set[str]] = set() # mutable-ok: local to this walk
merged: Final[list[object]] = [] # mutable-ok: local to this walk
for entry in entries:
names = guardrail_entry_names(entry)
if (names and seen.issuperset(names)) or (isinstance(entry, str) and entry in named_with_settings):
continue
merged.append(entry)
seen.update(names)
return merged
def guardrail_entry_names(entry: object) -> tuple[str, ...]:
if isinstance(entry, str):
return (entry,)
if isinstance(entry, dict):
return tuple(_GUARDRAIL_ENTRY_WITH_SETTINGS.validate_python(entry))
return tuple(cast("Mapping[str, object]", entry)) # cast-ok: a guardrail settings entry is keyed by name
return ()
def _settings_entry_names(entry: object) -> frozenset[str]:
return frozenset(guardrail_entry_names(entry)) if isinstance(entry, dict) else frozenset()
def _superseded_guardrail_entry(
entries: tuple[object, ...],
settings_names: tuple[frozenset[str], ...],
named_with_settings: frozenset[str],
index: int,
) -> bool:
entry: Final = entries[index]
if isinstance(entry, str):
return entry in named_with_settings or entry in entries[:index]
return bool(settings_names[index]) and settings_names[index] <= frozenset[str]().union(*settings_names[:index])
def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None = None) -> bool:
if event_type in _REALTIME_STREAMING_HOOKS:
return True

View file

@ -24,7 +24,7 @@ litellm/proxy/pass_through_endpoints/
The `PassThroughEndpointHandler` enables guardrails to run on passthrough endpoint requests by:
1. **Field Targeting**: Extracts specific fields from the request/response using JSONPath expressions configured in `request_fields` / `response_fields`
2. **Full Payload Fallback**: If no field targeting is configured, processes the entire payload
2. **Full Payload Fallback**: If no field targeting is configured, processes the entire request payload. A response is read whole, field names and integers included, except a provider's reply, where the ids, timestamps and usage counts around the text are left out
3. **Config Access**: Uses `get_passthrough_guardrails_config()` / `set_passthrough_guardrails_config()` helpers to access the passthrough guardrails configuration stored in request metadata
## Example Config

View file

@ -6,14 +6,15 @@ It uses the field targeting configuration from litellm_logging_obj
to extract specific fields for guardrail processing.
"""
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Optional, Protocol, runtime_checkable
from pydantic import TypeAdapter
from litellm._logging import verbose_proxy_logger
from litellm.integrations.custom_guardrail import without_server_streaming_classification
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, has_attributes
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -29,6 +30,95 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
# The fields of a provider's JSON reply that name or encode something rather than carry its text, by the field they sit
# under, and the fields that hold what the model wrote, which are read whole.
_REPLY_TOP_LEVEL_FIELDS: Final = frozenset(
(
"finish_reason",
"id",
"model",
"modelVersion",
"object",
"responseId",
"role",
"service_tier",
"status",
"stop_reason",
"stop_sequence",
"system_fingerprint",
"type",
)
)
_REPLY_NESTED_FIELDS: Final = MappingProxyType(
{
"candidates": frozenset(("finishReason",)),
"choices": frozenset(("finish_reason",)),
"content": frozenset(("id", "role", "type")),
"data": frozenset(("b64_json", "embedding", "object")),
"message": frozenset(("role",)),
"output": frozenset(("id", "role", "status", "type")),
"parts": frozenset(("thoughtSignature",)),
"predictions": frozenset(("bytesBase64Encoded",)),
"tool_calls": frozenset(("id", "type")),
}
)
_REPLY_MODEL_OUTPUT_FIELDS: Final = frozenset(("args", "arguments", "input", "parameters"))
_REPLY_LIST_FIELDS: Final = ("candidates", "choices", "output", "predictions")
_IMAGE_REPLY_FIELDS: Final = frozenset(("created", "data"))
_NO_FIELDS: Final[frozenset[str]] = frozenset()
def _is_provider_reply(response: Mapping[str, object]) -> bool:
"""Whether a JSON response has the shape of a provider's reply, by the type of the field that names it, since a
custom API's JSON can use the same field names."""
return (
any(isinstance(response.get(field), list) for field in _REPLY_LIST_FIELDS)
or isinstance(response.get("object"), str)
or (response.get("type") == "message" and "stop_reason" in response)
or ("finish_reason" in response and "message" in response)
or _IMAGE_REPLY_FIELDS.issubset(response)
)
def _reply_field_position(name: str, skipped: frozenset[str] | None) -> frozenset[str] | None:
if skipped is None or name in _REPLY_MODEL_OUTPUT_FIELDS:
return None
return _REPLY_NESTED_FIELDS.get(name, _NO_FIELDS)
def _reply_entries(
fields: Mapping[str, object], skipped: frozenset[str] | None
) -> Iterator[tuple[object, frozenset[str] | None]]:
for name, item in fields.items():
if skipped is None:
yield name, None
if skipped is None or name not in skipped:
yield item, _reply_field_position(name, skipped)
def _response_text(response: Mapping[str, object]) -> str:
"""The text a guardrail reads from a JSON response. In a provider's reply that is its strings without the ids and
encoded data around them, so a timestamp or a usage count doesn't read as a phone or card number. What the model
wrote into a tool call, and any other API's JSON, is read whole, field names and integers included. Floats are
scores and vectors."""
top_level: Final = _REPLY_TOP_LEVEL_FIELDS if _is_provider_reply(response) else None
pending: Final[list[tuple[object, frozenset[str] | None]]] = [(response, top_level)] # mutable-ok: stack
texts: Final[list[str]] = [] # mutable-ok: local to this walk
while pending:
value, skipped = pending.pop()
fields = as_str_mapping(value)
items = as_object_list(value)
if isinstance(value, str):
texts.append(value)
elif skipped is None and isinstance(value, int) and not isinstance(value, bool):
texts.append(str(value))
elif fields is not None:
pending.extend(reversed(tuple(_reply_entries(fields, skipped))))
elif items is not None:
pending.extend((item, skipped) for item in reversed(items))
return "\n".join(text for text in texts if text)
class PassThroughEndpointHandler(BaseTranslation):
"""
Handler for processing passthrough endpoint requests with guardrails.
@ -153,6 +243,7 @@ class PassThroughEndpointHandler(BaseTranslation):
litellm_logging_obj: Optional logging object
user_api_key_dict: User API key metadata to pass to guardrails
"""
response_fields: Final = as_str_mapping(response)
if not isinstance(response, dict):
verbose_proxy_logger.debug("PassThroughEndpointHandler: Response is not a dict, skipping")
return response
@ -168,7 +259,11 @@ class PassThroughEndpointHandler(BaseTranslation):
field_expressions: Final = settings.response_fields if settings else None
# Extract text to check
text_to_check: Final = self._extract_text_for_guardrail(response, field_expressions)
text_to_check: Final = (
self._extract_text_for_guardrail(response, field_expressions)
if field_expressions
else _response_text(response_fields or {})
)
if not text_to_check:
return response

View file

@ -365,7 +365,10 @@ class GenericGuardrailAPI(CustomGuardrail):
top_level_metadata: Final = request_data.get("metadata", {})
# Merge both sources, preferring litellm_metadata if both exist
metadata_dict: Final = {**top_level_metadata, **litellm_metadata}
metadata_dict: Final = {
**(top_level_metadata if isinstance(top_level_metadata, dict) else {}),
**(litellm_metadata if isinstance(litellm_metadata, dict) else {}),
}
if not metadata_dict:
return result_metadata

View file

@ -15,6 +15,7 @@ from typing import Final, Literal
import httpx
from fastapi import HTTPException
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -22,6 +23,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
httpxSpecialProvider,
@ -45,6 +47,20 @@ INPUT_POSITIONING_MAP: Final = {
Role.ASSISTANT.value: 2,
}
_CONTENT_PARTS: Final = TypeAdapter(list[object])
def _input_item_text(item: object) -> str:
"""The text of one entry of a list ``input``: a string, or a Responses message's string or parts ``content``."""
if isinstance(item, str):
return item
content: Final = (as_str_mapping(item) or {}).get("content")
if isinstance(content, str):
return content
parts: Final = _CONTENT_PARTS.validate_python(content) if isinstance(content, list) else ()
part_texts: Final = ((as_str_mapping(part) or {}).get("text") for part in parts)
return "\n".join(text for text in part_texts if isinstance(text, str))
class lakeraAI_Moderation(CustomGuardrail):
@classmethod
@ -219,7 +235,10 @@ class lakeraAI_Moderation(CustomGuardrail):
}
)
elif "input" in data and isinstance(data["input"], list):
text = "\n".join(data["input"])
text = "\n".join(filter(None, map(_input_item_text, data["input"])))
if not text:
verbose_proxy_logger.debug("Skipping lakera prompt injection, input has no text")
return
_json_data = json.dumps(
{
"input": text,
@ -227,6 +246,10 @@ class lakeraAI_Moderation(CustomGuardrail):
}
)
if not _json_data and call_type == "pass_through_endpoint":
verbose_proxy_logger.debug("Skipping lakera prompt injection, pass-through body has no messages or input")
return
verbose_proxy_logger.debug("Lakera AI Request Args %s", _json_data)
# https://platform.lakera.ai/account/api-keys

View file

@ -781,6 +781,9 @@ class LakeraAIGuardrail(CustomGuardrail):
# Use a copy of original_messages so _mask_pii_in_messages does not mutate data["messages"]
post_call_messages: Final = list(copy.deepcopy(original_messages)) + response_messages
if not post_call_messages:
verbose_proxy_logger.debug("Lakera AI: not running guardrail. No inspectable text in data")
return response
# Call Lakera guardrail
lakera_guardrail_response, _ = await self.call_v2_guard(

View file

@ -21,6 +21,7 @@ from litellm.integrations.custom_guardrail import (
CustomGuardrail,
log_guardrail_information,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.llms.custom_httpx.http_handler import (
@ -63,6 +64,11 @@ class _Action(str, enum.Enum):
GUARDRAIL_INTERVENED = "GUARDRAIL_INTERVENED"
def _user_api_key_alias(request_data: Mapping[str, object], metadata_key: str) -> object:
"""The key alias recorded under ``metadata_key``, which a pass-through client can set to anything."""
return (as_str_mapping(request_data.get(metadata_key)) or {}).get("user_api_key_alias")
class NomaV2Guardrail(CustomGuardrail):
reads_stream_marks: ClassVar[bool] = True
"""Noma keeps one record per streamed reply when every scan carries ``stream`` and only the last one carries the
@ -336,8 +342,8 @@ class NomaV2Guardrail(CustomGuardrail):
# (ports v1 fallback from PR #16832).
if application_id is None:
application_id = self._get_non_empty_str(
request_data.get("litellm_metadata", {}).get("user_api_key_alias")
) or self._get_non_empty_str(request_data.get("metadata", {}).get("user_api_key_alias"))
_user_api_key_alias(request_data, "litellm_metadata")
) or self._get_non_empty_str(_user_api_key_alias(request_data, "metadata"))
try:
payload: Final = self._build_scan_payload(

View file

@ -4377,11 +4377,12 @@ async def add_guardrails_from_policy_engine(
all_tags: Final = _policy_match_tags(data, metadata_variable_name)
_team_alias: Final = user_api_key_dict.team_alias
_key_alias: Final = user_api_key_dict.key_alias
_model: Final = data.get("model")
context: Final = PolicyMatchContext.model_validate(
{
"team_alias": _team_alias if isinstance(_team_alias, str) else None,
"key_alias": _key_alias if isinstance(_key_alias, str) else None,
"model": data.get("model"),
"model": _model if isinstance(_model, str) else None,
"tags": all_tags,
}
)

View file

@ -442,6 +442,7 @@ async def gemini_proxy_route(
custom_llm_provider="gemini",
is_streaming_request=is_streaming_request,
query_params=merged_params,
inherit_auth_guardrails=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value: Final = await endpoint_func(
request,
@ -1107,6 +1108,7 @@ async def anthropic_proxy_route(
target=str(updated_url),
custom_headers=_upstream_headers_for_anthropic_route(request, user_api_key_dict, auth_header),
is_streaming_request=is_streaming_request,
inherit_auth_guardrails=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value: Final = await endpoint_func(
request,
@ -3220,6 +3222,7 @@ async def _base_vertex_proxy_route(
upstream_headers_filter=partial(
_without_dropped_anthropic_betas, _vertex_dropped_anthropic_betas(general_settings, request.headers)
),
inherit_auth_guardrails=True,
) # dynamically construct pass-through endpoint based on incoming path
try:
@ -3822,6 +3825,7 @@ class BaseOpenAIPassThroughHandler:
if custom_llm_provider
else None
),
inherit_auth_guardrails=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value: Final = await endpoint_func(
request,

View file

@ -6,10 +6,19 @@ import posixpath
import re
import traceback
from base64 import b64encode
from collections.abc import AsyncGenerator, AsyncIterator, Callable, Iterable, Mapping, MutableMapping, Sequence
from collections.abc import (
AsyncGenerator,
AsyncIterator,
Awaitable,
Callable,
Iterable,
Mapping,
MutableMapping,
Sequence,
)
from dataclasses import dataclass
from datetime import datetime
from functools import reduce
from functools import partial, reduce
from itertools import chain, count, groupby, takewhile
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast
@ -57,7 +66,9 @@ from litellm.constants import (
)
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
guardrail_entry_names,
guardrail_request_data_with_streaming,
merged_guardrail_entries,
without_server_streaming_classification,
)
from litellm.integrations.custom_logger import CustomLogger
@ -113,6 +124,7 @@ from litellm.proxy.common_request_processing import (
open_sse_before_first_byte,
resolve_litellm_call_id,
)
from litellm.proxy.common_utils.callback_utils import strip_callback_config
from litellm.proxy.common_utils.error_body_call_id import JSON_OBJECT, error_body_call_id, with_call_id
from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports
_read_request_body, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export
@ -136,6 +148,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul
_key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy
_strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs
get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above
move_guardrails_to_metadata, # pyright: ignore[reportUnknownVariableType] # untyped data param; the same merge the native routes run
redact_credential_headers,
strip_client_guardrail_information,
)
@ -145,10 +158,11 @@ from litellm.proxy.route_priority import (
configured_pass_through_routes_first,
shadowed_by_builtin_claim,
)
from litellm.proxy.utils import normalize_route_for_root_path
from litellm.proxy.utils import normalize_route_for_root_path, pipeline_managed_guardrail_names
from litellm.repositories.team_repository import TeamRepository
from litellm.secret_managers.main import get_secret_str
from litellm.types import utils as types_utils
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.litellm_params import ProxyRequestState, wire_names
from litellm.types.llms.custom_http import httpxSpecialProvider
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
@ -169,7 +183,7 @@ from .llm_provider_handlers.tinyfish_passthrough_logging_handler import (
)
from .llm_provider_handlers.vertex_passthrough_logging_handler import VertexPassthroughLoggingHandler
from .streaming_handler import PassThroughStreamingHandler
from .success_handler import PassThroughEndpointLogging
from .success_handler import PassThroughEndpointLogging, finalize_cost_without_success_logs
from .upstream_usage_headers import (
UpstreamReportedUsage,
apply_upstream_reported_model,
@ -180,12 +194,17 @@ from .upstream_usage_headers import (
if TYPE_CHECKING:
from litellm.proxy.proxy_server import ProxyConfig
_GuardedResponse = TypeVar("_GuardedResponse")
router: Final = APIRouter()
pass_through_endpoint_logging: Final = PassThroughEndpointLogging()
_blocked_response_cost_logging: Final = PassThroughEndpointLogging(log_dispatch=finalize_cost_without_success_logs)
_METADATA_KEYS: Final = frozenset(("litellm_metadata", "metadata"))
_DECODED_JSON: Final = TypeAdapter(object)
_GUARDRAIL_ENTRIES: Final = TypeAdapter(list[object])
_POLICY_PIPELINE_FIELDS: Final = ("_guardrail_pipelines", "_pipeline_managed_guardrails")
_KEPT_OUT_OF_LITELLM_PARAMS: Final = _METADATA_KEYS | frozenset(wire_names(ProxyRequestState))
_PRICING_MODEL_OVERRIDE_PARAMS: Final = frozenset(("base_model",))
@ -867,7 +886,158 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
_update_stream_param_based_on_request_body = update_stream_param_based_on_request_body
def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | None) -> None:
async def _inherit_auth_guardrails(
parsed_body: dict[str, object] | None, # mutable-ok: merges the guardrails into the live request body
user_api_key_dict: UserAPIKeyAuth,
configured_guardrails: Sequence[object] | None,
inherit: bool,
) -> Sequence[object] | None:
"""Return the guardrails a pass-through request runs after the provider responds.
The endpoint's configured guardrails win. Without any, and with ``inherit`` and a JSON body, the key, team,
project, policy and request-body guardrails are merged into ``parsed_body["metadata"]`` for the pre-call hook,
with the ones the client named in ``metadata.guardrails`` kept after them, and returned only when at least one
of them runs post_call.
"""
if configured_guardrails or not inherit or not parsed_body:
return configured_guardrails
if not isinstance(parsed_body.get("metadata"), dict):
parsed_body["metadata"] = {} # rebind-ok: the merge below writes into the live request body's metadata
client_named: Final = (as_str_mapping(parsed_body["metadata"]) or {}).get("guardrails")
await move_guardrails_to_metadata(
data=parsed_body,
_metadata_variable_name="metadata",
user_api_key_dict=user_api_key_dict,
)
metadata: Final = parsed_body["metadata"]
merged: Final = (as_str_mapping(metadata) or {}).get("guardrails")
if not isinstance(merged, list):
return None
entries: Final = merged_guardrail_entries(
_GUARDRAIL_ENTRIES.validate_python(merged), _client_named_guardrail_entries(client_named)
)
if entries and isinstance(metadata, dict):
metadata["guardrails"] = entries
admin_scoped_body: Final = {
**parsed_body,
"metadata": {**(as_str_mapping(metadata) or {}), **_admin_guardrail_metadata(user_api_key_dict)},
}
return entries if entries and _named_guardrail_runs_post_call(admin_scoped_body, entries) else None
def _client_named_guardrail_entries(client_named: object) -> Sequence[object]:
"""The guardrails ``metadata.guardrails`` names: its entries, or the keys of the dict form the guardrails read"""
if isinstance(client_named, list):
return _GUARDRAIL_ENTRIES.validate_python(client_named)
return tuple(as_str_mapping(client_named) or ())
def _admin_guardrail_metadata(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object]:
"""The key and team metadata ``should_run_guardrail`` reads a global guardrail opt-out from, as the native routes set it"""
return MappingProxyType(
{
"user_api_key_metadata": strip_callback_config(user_api_key_dict.metadata),
"user_api_key_team_metadata": strip_callback_config(user_api_key_dict.team_metadata),
}
)
def _policy_pipeline_metadata(kwargs: Mapping[str, object], inherited: bool) -> Mapping[str, object]:
"""The policy pipelines resolved for a request that inherits auth guardrails, from the logging metadata that
request setup moved the body's ``metadata`` into"""
if not inherited:
return MappingProxyType({})
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) or {}
metadata: Final = as_str_mapping(litellm_params.get("metadata")) or {}
return {field: metadata[field] for field in _POLICY_PIPELINE_FIELDS if field in metadata}
def _starts_post_call_pass(guardrails: Sequence[object] | None, policy_pipelines: Mapping[str, object]) -> bool:
return bool(guardrails) or bool(pipeline_managed_guardrail_names({"metadata": dict(policy_pipelines)}, "post_call"))
def _post_call_hook_data(
parsed_body: Mapping[str, object] | None,
guardrails: Sequence[object] | None,
logging_obj: LiteLLMLoggingObj,
user_api_key_dict: UserAPIKeyAuth,
policy_pipelines: Mapping[str, object],
inherited: bool,
) -> dict[str, object]: # mutable-ok: the guardrails record their results in this request payload
"""The body with what the post-call guardrails read and the kwargs split stripped: ``metadata`` naming the guardrails,
which ``should_run_guardrail`` checks, the policy pipelines to run, and the logging object, which unified
(apply_guardrail) guardrails resolve a pass-through response's call type from when the route has no
API_ROUTE_TO_CALL_TYPES entry"""
body: Final = parsed_body or {}
admin_metadata: Final = _admin_guardrail_metadata(user_api_key_dict) if inherited else MappingProxyType({})
return {
**body,
"metadata": {
**(as_str_mapping(body.get("metadata")) or {}),
**policy_pipelines,
"guardrails": guardrails or (),
**admin_metadata,
},
"litellm_logging_obj": logging_obj,
}
def _provider_charge_settlement(
response: httpx.Response,
response_body: Mapping[str, object] | None,
url_route: str,
start_time: datetime,
logging_obj: LiteLLMLoggingObj,
parsed_body: Mapping[str, object] | None,
custom_llm_provider: str | None,
handler_kwargs: Mapping[str, object],
) -> Callable[[], Awaitable[None]]:
return partial(
_blocked_response_cost_logging.pass_through_async_success_handler,
finalize_before_dispatch=True,
httpx_response=response,
response_body=response_body,
url_route=url_route,
result="",
start_time=start_time,
end_time=datetime.now(start_time.tzinfo),
logging_obj=logging_obj,
cache_hit=False,
request_body=parsed_body or {},
custom_llm_provider=custom_llm_provider,
**handler_kwargs,
)
async def _run_post_call_guardrails_keeping_the_charge(
run_guardrails: Callable[[], Awaitable[_GuardedResponse]],
settle_provider_charge: Callable[[], Awaitable[None]],
) -> _GuardedResponse:
"""A guardrail that blocks the response, or fails, does so after the provider answered and billed the call, so the
charge is settled before the failure path writes its row, as on the native routes."""
try:
return await run_guardrails()
except Exception:
try:
await settle_provider_charge()
except Exception: # noqa: BLE001 - the guardrail's outcome still reaches the client
verbose_proxy_logger.exception("pass_through_endpoint: could not settle a blocked response's cost")
raise
def _named_guardrail_runs_post_call(data: Mapping[str, object], entries: Sequence[object]) -> bool:
names: Final = frozenset(chain.from_iterable(map(guardrail_entry_names, entries)))
return any(
guardrail.guardrail_name in names
and guardrail.should_run_guardrail( # pyright: ignore[reportUnknownMemberType] # takes the untyped request dict
data=data, event_type=GuardrailEventHooks.post_call
)
for guardrail in litellm.logging_callback_manager.get_custom_loggers_for_type(callback_type=CustomGuardrail)
if isinstance(guardrail, CustomGuardrail)
)
def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict[str, object] | None) -> None:
"""Copy guardrail logging entries from ``guardrail_data`` onto ``request_data``.
Post-call guardrails run against a throwaway ``hook_data`` dict (its
@ -881,13 +1051,16 @@ def _carry_guardrail_logging_info(request_data: dict, guardrail_data: dict | Non
if guardrail_data is None:
return
source_key: Final = get_metadata_variable_name_from_kwargs(guardrail_data)
source_metadata: Final = guardrail_data.get(source_key) or {}
source_metadata: Final = as_str_mapping(guardrail_data.get(source_key)) or {}
entries: Final = source_metadata.get("standard_logging_guardrail_information")
if not entries:
return
_, metadata = get_or_create_metadata_bucket(request_data)
metadata.setdefault("standard_logging_guardrail_information", list(entries))
metadata.setdefault(
"standard_logging_guardrail_information",
list(cast("Iterable[object]", entries)), # cast-ok: guardrails record a list of entries here
)
_GOOGLE_PASSTHROUGH_PROVIDERS: Final = frozenset(
@ -1255,6 +1428,7 @@ async def pass_through_request(
timeout: float | None = None,
upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers,
operation: GenAIOperation | None = None,
inherit_auth_guardrails: bool = False,
):
"""
Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called
@ -1277,6 +1451,8 @@ async def pass_through_request(
general_settings.pass_through_request_timeout, then 600s.
upstream_headers_filter: Optional last rewrite of the headers sent upstream, after x-pass- expansion
operation: Optional GenAI operation the route reports, unless the upstream names its own
inherit_auth_guardrails: Run the key, team, project and policy guardrails on a JSON body, as the
native routes do, when the pass-through guardrails config resolves to none
"""
from litellm.exceptions import ModifyResponseException
from litellm.litellm_core_utils.litellm_logging import Logging
@ -1385,6 +1561,13 @@ async def pass_through_request(
_parsed_body["metadata"]["guardrails"] = guardrails_to_run
verbose_proxy_logger.debug("Added guardrails to passthrough request metadata: %s", guardrails_to_run)
post_call_guardrails: Final = await _inherit_auth_guardrails(
parsed_body=cast(dict[str, object] | None, _parsed_body), # cast-ok: json
user_api_key_dict=user_api_key_dict,
configured_guardrails=guardrails_to_run,
inherit=inherit_auth_guardrails,
)
## LOGGING OBJECT ## - initialize before pre_call_hook so guardrails can access it
# Surface the requested model (when the body carries one) so logging/spans
# read e.g. ``chat gpt-4o`` instead of ``chat unknown``.
@ -1925,24 +2108,43 @@ async def pass_through_request(
logging_obj=logging_obj,
)
if response.status_code < 400 and response_body is not None and guardrails_to_run:
# Build an enriched data dict: _parsed_body has been stripped of
# `metadata` by both pre_call_hook and _init_kwargs_for_pass_through_endpoint,
# so we re-attach the configured guardrails here so should_run_guardrail
# sees them.
hook_data: Final = dict(_parsed_body or {})
existing_metadata = hook_data.get("metadata")
if not isinstance(existing_metadata, dict):
existing_metadata = {}
hook_data["metadata"] = {
**existing_metadata,
"guardrails": guardrails_to_run,
}
policy_pipelines: Final = _policy_pipeline_metadata(
cast(dict[str, object], kwargs), # cast-ok: the logging fields init_kwargs built above
inherit_auth_guardrails,
)
if (
response.status_code < 400
and response_body is not None
and _starts_post_call_pass(post_call_guardrails, policy_pipelines)
):
json_body: Final = cast(dict[str, object] | None, _parsed_body) # cast-ok: json
hook_data: Final = _post_call_hook_data(
json_body,
post_call_guardrails,
logging_obj,
user_api_key_dict,
policy_pipelines,
inherit_auth_guardrails,
)
post_call_guardrail_data = hook_data
response_body = await proxy_logging_obj.post_call_success_hook(
data=hook_data,
user_api_key_dict=user_api_key_dict,
response=response_body,
response_body = await _run_post_call_guardrails_keeping_the_charge(
partial(
proxy_logging_obj.post_call_success_hook,
data=hook_data,
user_api_key_dict=user_api_key_dict,
response=response_body,
guardrails_only=inherit_auth_guardrails,
),
_provider_charge_settlement(
response,
cast(dict[str, object] | None, response_body), # cast-ok: json
str(url),
start_time,
logging_obj,
json_body,
custom_llm_provider,
cast(dict[str, object], kwargs), # cast-ok: the logging fields init_kwargs built above
),
)
if isinstance(response_body, dict):
content = json.dumps(response_body).encode("utf-8")
@ -2320,6 +2522,7 @@ def create_pass_through_route(
timeout: float | None = None,
upstream_headers_filter: Callable[[Mapping[str, str]], Mapping[str, str]] = _unfiltered_headers,
operation: GenAIOperation | None = None,
inherit_auth_guardrails: bool = False,
):
# check if target is an adapter.py or a url
from litellm._uuid import uuid
@ -2469,6 +2672,7 @@ def create_pass_through_route(
timeout=cast(float | None, param_timeout),
upstream_headers_filter=upstream_headers_filter,
operation=operation,
inherit_auth_guardrails=inherit_auth_guardrails,
)
finally:
if hasattr(request.state, LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY):

View file

@ -79,6 +79,18 @@ def _websocket_frames(
)
async def finalize_cost_without_success_logs(
*,
logging_obj: LiteLLMLoggingObj,
standard_logging_response_object: object,
**_success_log_fields: object, # kwargs-ok: accepts every field a log dispatch receives; none are needed here
) -> None:
"""Settles the provider's charge on the logging object for a response that won't be logged as a success"""
from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost
await finalize_proxy_response_cost(logging_obj, standard_logging_response_object)
class PassThroughEndpointLogging:
def __init__(
self,

View file

@ -873,6 +873,29 @@ def _drop_response_steps_that_did_not_run(
_drop_applied_guardrails_in_place(data.get(bucket_key), not_run)
def _add_applied_guardrails_in_place(
data: dict[str, object], # mutable-ok: the names are recorded in the live request body's metadata
names: Sequence[str],
) -> None:
"""Adds each name the way add_guardrail_to_applied_guardrails_header does, in time linear in the names"""
from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
if not names:
return
metadata: Final = cast("dict[str, object]", get_or_create_metadata_bucket(data)[1]) # cast-ok: str-keyed bucket
if "applied_guardrails" not in metadata:
metadata["applied_guardrails"] = list(dict.fromkeys(names))
return
applied: Final = metadata["applied_guardrails"]
if not isinstance(applied, list):
for name in names:
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=name)
return
entries: Final = cast("list[object]", applied) # cast-ok: entries are only compared with names
present: Final = frozenset(name for name in entries if isinstance(name, str))
entries.extend(dict.fromkeys(name for name in names if name not in present))
def _drop_applied_guardrails_in_place(bucket: object, names: frozenset[str]) -> None:
if not isinstance(bucket, dict):
return
@ -2936,10 +2959,6 @@ class ProxyLogging:
policy pipeline steps that have not run. With ``skip_guardrails`` the pipelines run on
another request, so every step is added.
"""
from litellm.proxy.common_utils.callback_utils import (
add_guardrail_to_applied_guardrails_header,
)
metadata_standard: Final = data.get("metadata") or {}
metadata_litellm: Final = data.get("litellm_metadata") or {}
@ -2959,18 +2978,21 @@ class ProxyLogging:
if not isinstance(applied_guardrails, list):
applied_guardrails = []
not_run: Final = (
frozenset[str]()
if skip_guardrails
else _request_steps_that_have_not_run(cast(dict[str, object], data)) # cast-ok: request payload
request: Final = cast("dict[str, object]", data) # cast-ok: request payload
not_run: Final = frozenset[str]() if skip_guardrails else _request_steps_that_have_not_run(request)
already_applied: Final = frozenset(
name
for name in cast("Sequence[object]", applied_guardrails) # cast-ok: checked to be a list above
if isinstance(name, str)
)
_add_applied_guardrails_in_place(
request,
[
name
for name in guardrails_in_metadata
if isinstance(name, str) and name not in already_applied and name not in not_run
],
)
for guardrail_name in guardrails_in_metadata:
if (
isinstance(guardrail_name, str)
and guardrail_name not in applied_guardrails
and guardrail_name not in not_run
):
add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name=guardrail_name)
async def _maybe_execute_pipelines(
self,
@ -4381,6 +4403,7 @@ class ProxyLogging:
data: dict,
response: LLMResponseTypes,
user_api_key_dict: UserAPIKeyAuth,
guardrails_only: bool = False,
):
"""
Allow user to modify outgoing data
@ -4390,6 +4413,10 @@ class ProxyLogging:
2. /embeddings
3. /image/generation
4. /files
``guardrails_only`` is for a relayed response that never reached a router deployment and isn't in the shape the
other hooks expect: only the guardrails the request selected run, not the guardrails of a model that merely
shares the body's ``model`` name, nor the callbacks that aren't guardrails.
"""
from litellm.proxy.proxy_server import llm_router
@ -4407,7 +4434,9 @@ class ProxyLogging:
guardrail_callbacks, other_callbacks = _partition_post_call_callbacks()
try:
# Merge model-level guardrails before checking which guardrails to run
guardrail_data: Final = check_and_merge_model_level_guardrails(data=data, llm_router=llm_router)
guardrail_data: Final = (
data if guardrails_only else check_and_merge_model_level_guardrails(data=data, llm_router=llm_router)
)
parallel_guardrails: Final[tuple[CustomGuardrail, ...]] = tuple(
callback
@ -4475,7 +4504,7 @@ class ProxyLogging:
############ Handle CustomLogger ###############################
#################################################################
for callback in other_callbacks:
for callback in () if guardrails_only else other_callbacks:
callback_response: LLMResponseTypes | None = await callback.async_post_call_success_hook(
user_api_key_dict=user_api_key_dict, data=data, response=response
)

View file

@ -0,0 +1,691 @@
import json
import uuid
from collections.abc import Iterator, Mapping
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final, Literal
import httpx
import pytest
import yaml
from integration._support.anthropic_sse import message_json
from integration._support.client import Gateway, Scenario, eventually
from integration._support.openai_wire import chat_reply
from integration._support.process import OwnedProxy, owned_proxy_process
from integration._support.safeguards_rig import key_hash
from integration._support.wire import Reply, Request, Wire, wire_server
from pydantic import JsonValue, TypeAdapter
GUARDRAIL: Final = "passthrough-auth-guardrail"
GLOBAL_POST_CALL: Final = "passthrough-global-post-call"
KEY_POST_CALL: Final = "passthrough-key-post-call"
GENERIC_POST_CALL: Final = "passthrough-generic-post-call"
BLOCKING_POST_CALL: Final = "passthrough-post-call-blocks-every-reply"
MODEL_POST_CALL: Final = "passthrough-model-post-call"
GUARDED_MODEL: Final = "passthrough-guarded-model"
POST_CALL_GUARDED_MODEL: Final = "passthrough-post-call-guarded-model"
BLOCK_MARKER: Final = "block-this-prompt"
RAISING_HOOK_MODEL: Final = "passthrough-model-a-logger-hook-cannot-read"
_RAISING_HOOK_MODULE: Final = f"""
from litellm.integrations.custom_logger import CustomLogger
class RaisingHook(CustomLogger):
async def async_post_call_success_hook(self, data, user_api_key_dict, response):
if data.get("model") == "{RAISING_HOOK_MODEL}":
raise ValueError("this hook only reads native responses")
proxy_handler_instance = RaisingHook()
"""
OPENAI_ROUTE: Final = "/openai/v1/chat/completions"
ANTHROPIC_ROUTE: Final = "/anthropic/v1/messages"
GEMINI_ROUTE: Final = "/gemini/v1beta/models/gemini-2.5-flash:generateContent"
CUSTOM_ROUTE: Final = "/passthrough-auth-custom"
PHONE_POST_CALL: Final = "passthrough-auth-phone-post-call"
PIPELINE_BLOCKS: Final = "passthrough-pipeline-blocks-the-response"
PIPELINE_ALLOWS: Final = "passthrough-pipeline-allows-the-response"
PIPELINE_BEFORE_THE_CALL: Final = "passthrough-pipeline-before-the-call"
_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
_JSON: Final = TypeAdapter(dict[str, JsonValue])
_JSON_LIST: Final = TypeAdapter(list[JsonValue])
Level = Literal["key", "team", "project"]
def _route_body(route: str, prompt: str) -> dict[str, JsonValue]:
match route:
case "/anthropic/v1/messages":
return {"model": "claude-sonnet-5-5", "max_tokens": 16, "messages": [{"role": "user", "content": prompt}]}
case "/gemini/v1beta/models/gemini-2.5-flash:generateContent":
return {"contents": [{"role": "user", "parts": [{"text": prompt}]}]}
case _:
return {"model": "gpt-5.6", "messages": [{"role": "user", "content": prompt}]}
def _pipeline_policy(mode: str, guardrail: str, on_fail: str) -> dict[str, JsonValue]:
return {
"guardrails": {"add": [guardrail]},
"pipeline": {"mode": mode, "steps": [{"guardrail": guardrail, "on_pass": "allow", "on_fail": on_fail}]},
}
def _verdict(request: Request) -> Reply:
action: Final = "BLOCKED" if BLOCK_MARKER.encode() in request.body else "NONE"
return Reply(body=json.dumps({"action": action, "blocked_reason": "synthetic block"}).encode())
def _block_every_reply(_request: Request) -> Reply:
return Reply(body=json.dumps({"action": "BLOCKED", "blocked_reason": "synthetic block"}).encode())
_OPENAI_BODY: Final = json.dumps(
{
**_JSON.validate_json(chat_reply("chatcmpl-passthrough-auth", "gpt-5.6", "Hi", stream=False).body),
"created": 1793600123,
},
indent=2,
).encode()
def _openai_reply(_request: Request) -> Reply:
return Reply(body=_OPENAI_BODY)
def _lakera_clean(_request: Request) -> Reply:
return Reply(body=json.dumps({"flagged": False, "breakdown": [], "payload": []}).encode())
def _lakera_flagged(_request: Request) -> Reply:
return Reply(
body=json.dumps(
{
"flagged": True,
"breakdown": [{"detector_type": "prompt_attack", "detected": True, "message_id": 0}],
"payload": [],
}
).encode()
)
_ANTHROPIC_BODY: Final = json.dumps(
_JSON.validate_json(message_json("msg_passthrough_auth", "claude-sonnet-5-5", "Hi")), indent=2
).encode()
_GEMINI_BODY: Final = json.dumps(
{
"candidates": [{"content": {"role": "model", "parts": [{"text": "Hi"}]}, "finishReason": "STOP"}],
"usageMetadata": {"promptTokenCount": 4, "candidatesTokenCount": 1, "totalTokenCount": 5},
},
indent=2,
).encode()
_UPSTREAM_BODIES: Final = MappingProxyType(
{OPENAI_ROUTE: _OPENAI_BODY, ANTHROPIC_ROUTE: _ANTHROPIC_BODY, GEMINI_ROUTE: _GEMINI_BODY}
)
def _anthropic_reply(_request: Request) -> Reply:
return Reply(body=_ANTHROPIC_BODY)
def _gemini_reply(_request: Request) -> Reply:
return Reply(body=_GEMINI_BODY)
@dataclass(frozen=True, slots=True)
class AuthGuardrailRig:
owned: OwnedProxy
vendor: Wire
post_call_vendor: Wire
key_post_call_vendor: Wire
generic_post_call_vendor: Wire
model_post_call_vendor: Wire
upstreams: Mapping[str, Wire]
@pytest.fixture(scope="module")
def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[AuthGuardrailRig]:
directory: Final = tmp_path_factory.mktemp("passthrough-auth-guardrails")
with ExitStack() as stack:
vendor: Final = stack.enter_context(wire_server(_verdict))
post_call_vendor: Final = stack.enter_context(wire_server(_lakera_clean))
key_post_call_vendor: Final = stack.enter_context(wire_server(_lakera_flagged))
generic_post_call_vendor: Final = stack.enter_context(wire_server(_verdict))
blocking_post_call_vendor: Final = stack.enter_context(wire_server(_block_every_reply))
model_post_call_vendor: Final = stack.enter_context(wire_server(_verdict))
upstreams: Final = MappingProxyType(
{
OPENAI_ROUTE: stack.enter_context(wire_server(_openai_reply)),
ANTHROPIC_ROUTE: stack.enter_context(wire_server(_anthropic_reply)),
GEMINI_ROUTE: stack.enter_context(wire_server(_gemini_reply)),
}
)
config: Final = _JSON.validate_python(yaml.safe_load(_CONFIG.read_text()))
general_settings: Final = _JSON.validate_python(config.get("general_settings") or {})
(directory / "passthrough_raising_hook.py").write_text(_RAISING_HOOK_MODULE)
path: Final = directory / "passthrough-auth-guardrails.yaml"
path.write_text(
yaml.safe_dump(
{
**config,
"litellm_settings": {
**_JSON.validate_python(config.get("litellm_settings") or {}),
"callbacks": ["passthrough_raising_hook.proxy_handler_instance"],
},
"general_settings": {
**general_settings,
"pass_through_endpoints": [
{
"path": CUSTOM_ROUTE,
"target": f"{upstreams[OPENAI_ROUTE].url}/v1/chat/completions",
"auth": True,
}
],
},
"policies": {
PIPELINE_BLOCKS: _pipeline_policy("post_call", KEY_POST_CALL, "block"),
PIPELINE_ALLOWS: _pipeline_policy("post_call", KEY_POST_CALL, "allow"),
PIPELINE_BEFORE_THE_CALL: _pipeline_policy("pre_call", GUARDRAIL, "block"),
},
"policy_attachments": [
{"policy": policy, "tags": [policy]}
for policy in (PIPELINE_BLOCKS, PIPELINE_ALLOWS, PIPELINE_BEFORE_THE_CALL)
],
"model_list": [
*_JSON_LIST.validate_python(config.get("model_list") or []),
{
"model_name": GUARDED_MODEL,
"litellm_params": {
"model": "openai/gpt-5.6",
"api_key": "synthetic-openai-key",
"guardrails": [GUARDRAIL],
},
},
{
"model_name": POST_CALL_GUARDED_MODEL,
"litellm_params": {
"model": "openai/gpt-5.6",
"api_key": "synthetic-openai-key",
"guardrails": [MODEL_POST_CALL],
},
},
],
"guardrails": [
{
"guardrail_name": GLOBAL_POST_CALL,
"litellm_params": {
"guardrail": "lakera_v2",
"mode": "post_call",
"default_on": True,
"api_base": post_call_vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": GUARDRAIL,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "pre_call",
"default_on": False,
"api_base": vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": KEY_POST_CALL,
"litellm_params": {
"guardrail": "lakera_v2",
"mode": "post_call",
"default_on": False,
"api_base": key_post_call_vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": BLOCKING_POST_CALL,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "post_call",
"default_on": False,
"api_base": blocking_post_call_vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": GENERIC_POST_CALL,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "post_call",
"default_on": False,
"api_base": generic_post_call_vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
{
"guardrail_name": PHONE_POST_CALL,
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "post_call",
"default_on": False,
"patterns": [
{"pattern_type": "prebuilt", "pattern_name": "us_phone", "action": "BLOCK"}
],
},
},
{
"guardrail_name": MODEL_POST_CALL,
"litellm_params": {
"guardrail": "generic_guardrail_api",
"mode": "post_call",
"default_on": False,
"api_base": model_post_call_vendor.url,
"api_key": "synthetic-guardrail-key",
},
},
],
}
)
)
environment: Final = {
"OPENAI_API_BASE": upstreams[OPENAI_ROUTE].url,
"OPENAI_API_KEY": "synthetic-openai-key",
"ANTHROPIC_API_BASE": upstreams[ANTHROPIC_ROUTE].url,
"ANTHROPIC_API_KEY": "synthetic-anthropic-key",
"GEMINI_API_BASE": upstreams[GEMINI_ROUTE].url,
"GEMINI_API_KEY": "synthetic-gemini-key",
}
owned: Final = stack.enter_context(owned_proxy_process(session_gateway, directory, environment, config=path))
yield AuthGuardrailRig(
owned,
vendor,
post_call_vendor,
key_post_call_vendor,
generic_post_call_vendor,
model_post_call_vendor,
upstreams,
)
def _key_with_guardrail(scenario: Scenario, level: Level) -> str:
attached: Final[dict[str, JsonValue]] = {"metadata": {"guardrails": [GUARDRAIL]}}
team: Final = scenario.team(**(attached if level == "team" else {}))
match level:
case "project":
return scenario.key(team_id=team, project_id=scenario.project(team, **attached))
case "key":
return scenario.key(team_id=team, **attached)
case _:
return scenario.key(team_id=team)
def _send(gateway: Gateway, route: str, body: Mapping[str, JsonValue], key: str) -> httpx.Response:
return gateway.request("POST", route, body, key=key, headers={"x-goog-api-key": key})
def _drain(rig: AuthGuardrailRig) -> None:
rig.vendor.drain()
rig.post_call_vendor.drain()
rig.key_post_call_vendor.drain()
rig.generic_post_call_vendor.drain()
rig.model_post_call_vendor.drain()
for upstream in rig.upstreams.values():
upstream.drain()
@pytest.mark.parametrize("level", ["key", "team", "project"])
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_auth_guardrail_blocks_the_prompt_before_the_provider(rig: AuthGuardrailRig, route: str, level: Level) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, level)
_drain(rig)
response: Final = _send(gateway, route, _route_body(route, BLOCK_MARKER), key)
assert response.status_code == 400, response.text
assert [BLOCK_MARKER.encode() in request.body for request in rig.vendor.drain()] == [True]
assert rig.upstreams[route].drain() == ()
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_auth_guardrail_lets_a_clean_prompt_reach_the_provider(rig: AuthGuardrailRig, route: str) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, "key")
marker: Final = f"clean-{uuid.uuid4().hex}"
_drain(rig)
response: Final = _send(gateway, route, _route_body(route, marker), key)
assert response.status_code == 200, response.text
assert [marker.encode() in request.body for request in rig.vendor.drain()] == [True]
assert [marker.encode() in request.body for request in rig.upstreams[route].drain()] == [True]
def test_empty_request_guardrails_list_does_not_drop_the_key_guardrail(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, "key")
_drain(rig)
response: Final = gateway.request(
"POST", OPENAI_ROUTE, {**_route_body(OPENAI_ROUTE, BLOCK_MARKER), "guardrails": []}, key=key
)
assert response.status_code == 400, response.text
assert rig.upstreams[OPENAI_ROUTE].drain() == ()
def test_request_without_a_body_is_not_sent_to_the_guardrail(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, "key")
_drain(rig)
response: Final = gateway.request("GET", "/openai/v1/models", key=key)
assert response.status_code == 200, response.text
assert rig.vendor.drain() == ()
assert len(rig.upstreams[OPENAI_ROUTE].drain()) == 1
def test_pre_call_guardrail_relays_the_provider_reply_untouched(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, "key")
_drain(rig)
response: Final = _send(gateway, OPENAI_ROUTE, _route_body(OPENAI_ROUTE, "What is the capital of France?"), key)
assert response.status_code == 200, response.text
assert response.content == _OPENAI_BODY
assert len(rig.vendor.drain()) == 1
assert rig.post_call_vendor.drain() == ()
def test_key_post_call_guardrail_runs_on_a_model_with_its_own_guardrails(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [KEY_POST_CALL]})
_drain(rig)
response: Final = _send(
gateway,
OPENAI_ROUTE,
{**_route_body(OPENAI_ROUTE, "What is the capital of France?"), "model": GUARDED_MODEL},
key,
)
assert response.status_code == 400, response.text
assert len(rig.key_post_call_vendor.drain()) == 1
assert len(rig.upstreams[OPENAI_ROUTE].drain()) == 1
@pytest.mark.parametrize("route", [ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_pre_call_guardrail_relays_every_provider_reply_untouched(rig: AuthGuardrailRig, route: str) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_guardrail(scenario, "team")
_drain(rig)
response: Final = _send(gateway, route, _route_body(route, "What is the capital of France?"), key)
assert response.status_code == 200, response.text
assert response.content == _UPSTREAM_BODIES[route]
assert len(rig.vendor.drain()) == 1
assert rig.post_call_vendor.drain() == ()
def test_custom_pass_through_endpoint_keeps_its_opt_in(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(
team_id=scenario.team(),
metadata={"guardrails": [GUARDRAIL], "allowed_passthrough_routes": [CUSTOM_ROUTE]},
)
_drain(rig)
response: Final = gateway.request("POST", CUSTOM_ROUTE, _route_body(OPENAI_ROUTE, BLOCK_MARKER), key=key)
assert response.status_code == 200, response.text
assert rig.vendor.drain() == ()
assert [BLOCK_MARKER.encode() in request.body for request in rig.upstreams[OPENAI_ROUTE].drain()] == [True]
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_key_apply_guardrail_post_call_scans_the_provider_reply(rig: AuthGuardrailRig, route: str) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [GENERIC_POST_CALL]})
_drain(rig)
response: Final = _send(gateway, route, _route_body(route, "What is the capital of France?"), key)
assert response.status_code == 200, response.text
assert [b'"Hi' in request.body for request in rig.generic_post_call_vendor.drain()] == [True]
def test_post_call_guardrail_does_not_read_a_reply_timestamp_as_a_phone_number(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [PHONE_POST_CALL]})
_drain(rig)
response: Final = _send(gateway, OPENAI_ROUTE, _route_body(OPENAI_ROUTE, "What is the capital of France?"), key)
assert response.status_code == 200, response.text
assert response.json()["created"] == 1793600123
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_guardrail_the_client_names_in_metadata_still_runs_for_a_guarded_key(rig: AuthGuardrailRig, route: str) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [GENERIC_POST_CALL]})
_drain(rig)
response: Final = _send(
gateway, route, {**_route_body(route, BLOCK_MARKER), "metadata": {"guardrails": [GUARDRAIL]}}, key
)
assert response.status_code == 400, response.text
assert [BLOCK_MARKER.encode() in request.body for request in rig.vendor.drain()] == [True]
assert rig.upstreams[route].drain() == ()
@pytest.mark.parametrize("settings", [{}, True])
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE, GEMINI_ROUTE])
def test_guardrail_the_client_names_in_the_dict_form_still_runs_for_a_guarded_key(
rig: AuthGuardrailRig, route: str, settings: JsonValue
) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [GENERIC_POST_CALL]})
_drain(rig)
response: Final = _send(
gateway, route, {**_route_body(route, BLOCK_MARKER), "metadata": {"guardrails": {GUARDRAIL: settings}}}, key
)
assert response.status_code == 400, response.text
assert [BLOCK_MARKER.encode() in request.body for request in rig.vendor.drain()] == [True]
assert rig.upstreams[route].drain() == ()
def _key_with_opt_out(scenario: Scenario, opt_out: str) -> str:
guarded: Final[dict[str, JsonValue]] = {"guardrails": [GENERIC_POST_CALL]}
match opt_out:
case "key_disable":
return scenario.key(metadata=guarded, disable_global_guardrails=True)
case "key_opted_out":
return scenario.key(metadata={**guarded, "opted_out_global_guardrails": [GLOBAL_POST_CALL]})
case "team_disable":
return scenario.key(team_id=scenario.team(disable_global_guardrails=True), metadata=guarded)
case "team_opted_out":
return scenario.key(
team_id=scenario.team(metadata={"opted_out_global_guardrails": [GLOBAL_POST_CALL]}), metadata=guarded
)
case _:
return scenario.key(metadata=guarded)
@pytest.mark.parametrize(
("opt_out", "global_vendor_calls"),
[("key_disable", 0), ("key_opted_out", 0), ("team_disable", 0), ("team_opted_out", 0), ("none", 1)],
)
@pytest.mark.parametrize("route", [OPENAI_ROUTE, "/v1/chat/completions"])
def test_post_call_pass_honors_a_global_guardrail_opt_out_as_the_native_route_does(
rig: AuthGuardrailRig, route: str, opt_out: str, global_vendor_calls: int
) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = _key_with_opt_out(scenario, opt_out)
_drain(rig)
response: Final = _send(
gateway,
route,
{**_route_body(OPENAI_ROUTE, "What is the capital of France?"), "model": GUARDED_MODEL},
key,
)
assert response.status_code == 200, response.text
assert len(rig.generic_post_call_vendor.drain()) == 1
assert len(rig.post_call_vendor.drain()) == global_vendor_calls
def test_model_guardrail_does_not_run_on_pass_through_traffic_that_names_the_model(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [GENERIC_POST_CALL]})
_drain(rig)
response: Final = _send(
gateway,
OPENAI_ROUTE,
{**_route_body(OPENAI_ROUTE, "What is the capital of France?"), "model": POST_CALL_GUARDED_MODEL},
key,
)
assert response.status_code == 200, response.text
assert len(rig.generic_post_call_vendor.drain()) == 1
assert rig.model_post_call_vendor.drain() == ()
def test_post_call_pass_leaves_a_logger_hook_for_native_responses_out(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [GENERIC_POST_CALL]})
_drain(rig)
response: Final = _send(
gateway,
OPENAI_ROUTE,
{**_route_body(OPENAI_ROUTE, "What is the capital of France?"), "model": RAISING_HOOK_MODEL},
key,
)
assert response.status_code == 200, response.text
assert len(rig.generic_post_call_vendor.drain()) == 1
assert len(rig.upstreams[OPENAI_ROUTE].drain()) == 1
def _charged_rows(gateway: Gateway, key: str) -> tuple[Mapping[str, object], ...]:
return tuple(
eventually(
lambda: gateway.rows(
'SELECT status, spend, total_tokens FROM "LiteLLM_SpendLogs" WHERE api_key = %s', (key_hash(key),)
),
bool,
)
)
def _tagged_body(route: str, tag: str) -> dict[str, JsonValue]:
return {**_route_body(route, "What is the capital of France?"), "metadata": {"tags": [tag]}}
@pytest.mark.parametrize("route", [OPENAI_ROUTE, ANTHROPIC_ROUTE])
def test_post_call_policy_pipeline_blocks_a_flagged_pass_through_response(rig: AuthGuardrailRig, route: str) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key()
_drain(rig)
response: Final = _send(gateway, route, _tagged_body(route, PIPELINE_BLOCKS), key)
assert response.status_code == 400, response.text
assert len(rig.key_post_call_vendor.drain()) == 1
assert len(rig.upstreams[route].drain()) == 1
assert [row["status"] for row in _charged_rows(gateway, key)] == ["failure"]
def test_post_call_policy_pipeline_decides_what_a_flagged_step_does(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key()
_drain(rig)
response: Final = _send(gateway, OPENAI_ROUTE, _tagged_body(OPENAI_ROUTE, PIPELINE_ALLOWS), key)
assert response.status_code == 200, response.text
assert len(rig.key_post_call_vendor.drain()) == 1
def test_pre_call_policy_pipeline_relays_the_provider_bytes(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key()
_drain(rig)
response: Final = _send(gateway, OPENAI_ROUTE, _tagged_body(OPENAI_ROUTE, PIPELINE_BEFORE_THE_CALL), key)
assert response.status_code == 200, response.text
assert response.content == _OPENAI_BODY
assert len(rig.vendor.drain()) == 1
def test_lakera_post_call_has_nothing_to_read_on_a_gemini_body(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [KEY_POST_CALL]})
_drain(rig)
response: Final = _send(gateway, GEMINI_ROUTE, _route_body(GEMINI_ROUTE, "What is the capital of France?"), key)
assert response.status_code == 200, response.text
assert rig.key_post_call_vendor.drain() == ()
assert len(rig.upstreams[GEMINI_ROUTE].drain()) == 1
def test_post_call_policy_pipeline_lakera_step_has_nothing_to_read_on_a_gemini_body(rig: AuthGuardrailRig) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key()
_drain(rig)
response: Final = _send(gateway, GEMINI_ROUTE, _tagged_body(GEMINI_ROUTE, PIPELINE_BLOCKS), key)
assert response.status_code == 200, response.text
assert rig.key_post_call_vendor.drain() == ()
assert len(rig.upstreams[GEMINI_ROUTE].drain()) == 1
@pytest.mark.parametrize(("route", "guardrail"), [(ANTHROPIC_ROUTE, KEY_POST_CALL), (GEMINI_ROUTE, BLOCKING_POST_CALL)])
def test_response_a_post_call_guardrail_blocks_still_charges_the_provider_call(
rig: AuthGuardrailRig, route: str, guardrail: str
) -> None:
gateway: Final = rig.owned.gateway
with gateway.scenario() as scenario:
key: Final = scenario.key(metadata={"guardrails": [guardrail]})
_drain(rig)
response: Final = _send(gateway, route, _route_body(route, "What is the capital of France?"), key)
assert response.status_code == 400, response.text
assert len(rig.upstreams[route].drain()) == 1
rows: Final = _charged_rows(gateway, key)
assert [row["status"] for row in rows] == ["failure"], rows
assert float(str(rows[0]["spend"])) > 0, rows
assert int(str(rows[0]["total_tokens"])) > 0, rows

View file

@ -5,7 +5,7 @@ import json
import pickle
import threading
from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, patch
import pytest
@ -3802,6 +3802,30 @@ def test_default_on_guardrail() -> None:
)
def test_merged_guardrail_entries_dedupes_an_all_plain_list():
from litellm.integrations.custom_guardrail import merged_guardrail_entries
names: Final = [f"g{index % 500}" for index in range(1000)]
assert merged_guardrail_entries(names) == [f"g{index}" for index in range(500)]
def test_merged_guardrail_entries_reads_each_entry_of_a_mixed_list_a_fixed_number_of_times():
from litellm.integrations import custom_guardrail
entries: Final = [*(f"g{index}" for index in range(1000)), {"g1": {"threshold": 1}}]
with patch.object(
custom_guardrail, "guardrail_entry_names", wraps=custom_guardrail.guardrail_entry_names
) as entry_names:
merged: Final = custom_guardrail.merged_guardrail_entries(entries)
assert "g1" not in merged
assert merged[-1] == {"g1": {"threshold": 1}}
assert len(merged) == 1000
assert entry_names.call_count <= 2 * len(entries)
def test_merged_guardrail_entries_names_each_guardrail_once_preferring_its_first_entry_with_settings():
from litellm.integrations.custom_guardrail import merged_guardrail_entries

View file

@ -1,5 +1,6 @@
import json
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock
import pytest
from typing_extensions import ReadOnly, TypedDict
@ -226,3 +227,123 @@ def test_passthrough_route_block_before_any_invoke_frame_is_a_whole_message_in_c
"message_stop",
]
assert events[2]["delta"]["text"] == "blocked"
async def _scanned_response_texts(response: dict[str, object]) -> tuple[str, ...]:
guardrail: Final = MagicMock(guardrail_name="post-call", apply_guardrail=AsyncMock())
await PassThroughEndpointHandler().process_output_response(response, guardrail, request_data={})
return tuple(text for call in guardrail.apply_guardrail.await_args_list for text in call.kwargs["inputs"]["texts"])
@pytest.mark.asyncio
async def test_post_call_scan_reads_the_reply_text_and_not_the_numbers_around_it():
response: Final = {
"id": "chatcmpl-1793600123",
"object": "chat.completion",
"created": 1793600123,
"model": "gpt-5.6",
"choices": [{"index": 0, "message": {"role": "assistant", "content": "Paris."}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 4111111111111111, "total_tokens": 5551234567},
}
assert await _scanned_response_texts(response) == ("Paris.",)
@pytest.mark.asyncio
async def test_post_call_scan_still_reads_every_string_a_reply_carries():
response: Final = {
"choices": [
{"message": {"content": "call 555-123-4567", "tool_calls": [{"function": {"arguments": '{"a": "b"}'}}]}}
],
"candidates": [{"content": {"parts": [{"text": "card 4111 1111 1111 1111"}]}}],
}
assert await _scanned_response_texts(response) == ('call 555-123-4567\n{"a": "b"}\ncard 4111 1111 1111 1111',)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
[
{"candidates": [{"content": {"parts": [{"functionCall": {"args": {"id": "alice@example.com"}}}]}}]},
{"content": [{"input": {"id": "alice@example.com", "message": {"role": "alice@example.com"}}}]},
{"choices": [{"message": {"tool_calls": [{"function": {"nested": {"type": "alice@example.com"}}}]}}]},
{"id": "alice@example.com", "result": {"status": "alice@example.com"}},
],
)
async def test_post_call_scan_reads_a_field_named_like_the_envelope_in_what_the_model_or_api_wrote(response):
assert all("alice@example.com" in text for text in await _scanned_response_texts(response))
assert await _scanned_response_texts(response) != ()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response, expected",
[
({"result": {"credit_card": 4111111111111111, "score": 0.4111111111111111, "ok": True}}, "4111111111111111"),
(
{"candidates": [{"content": {"parts": [{"functionCall": {"args": {"card": 4111111111111111}}}]}}]},
"4111111111111111",
),
({"choices": [{"message": {"tool_calls": [{"function": {"arguments": {"n": 5551234567}}}]}}]}, "5551234567"),
({"usage": {}, "customer": {"phone": 5551234567}}, "5551234567"),
({"output": {"card": 4111111111111111}}, "4111111111111111"),
({"type": "message", "phone": 5551234567}, "5551234567"),
],
)
async def test_post_call_scan_reads_an_integer_in_what_the_model_or_api_wrote(response, expected):
assert any(expected in text for text in await _scanned_response_texts(response))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
[
{"results": {"alice@example.com": "active"}},
{"candidates": [{"content": {"parts": [{"functionCall": {"args": {"alice@example.com": "active"}}}]}}]},
],
)
async def test_post_call_scan_reads_a_field_name_in_what_the_model_or_api_wrote(response):
assert any("alice@example.com" in text for text in await _scanned_response_texts(response))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response",
[
{"created": 1793600123, "data": [{"url": "https://x.example/a.png"}]},
{"id": "batch_1", "object": "batch", "created_at": 1793600123, "request_counts": {"total": 5551234567}},
{"predictions": [{"embeddings": {"values": [0.0123456789]}, "total": 5551234567}]},
{"finish_reason": "COMPLETE", "message": {"role": "assistant"}, "usage": {"total": 5551234567}},
],
)
async def test_post_call_scan_leaves_out_the_numbers_of_a_provider_reply(response):
assert all(
"1793600123" not in text and "5551234567" not in text for text in await _scanned_response_texts(response)
)
@pytest.mark.asyncio
async def test_post_call_scan_reads_only_the_text_of_an_anthropic_message():
response: Final = {
"id": "msg_5551234567",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [
{"type": "text", "text": "Paris."},
{"type": "tool_use", "id": "toolu_5551234567", "name": "lookup", "input": {"card": 4111111111111111}},
],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 5551234567, "output_tokens": 1},
}
assert await _scanned_response_texts(response) == ("Paris.\nlookup\ncard\n4111111111111111",)
@pytest.mark.asyncio
async def test_post_call_scan_skips_a_reply_with_no_text():
response: Final = {"object": "list", "data": [{"object": "embedding", "embedding": [0.4111111111111111]}]}
assert await _scanned_response_texts(response) == ()

View file

@ -2692,3 +2692,14 @@ async def test_during_mcp_call_blocked_raises() -> None:
)
assert exc_info.value.message == "credential detected in tool input"
@pytest.mark.parametrize("value", (None, 0, False, "", [], "text", 7))
def test_extract_user_api_key_metadata_reads_past_a_litellm_metadata_that_is_not_an_object(value: object) -> None:
guardrail: Final = GenericGuardrailAPI(api_base="http://vendor.invalid", guardrail_name="generic")
metadata: Final = guardrail._extract_user_api_key_metadata(
{"metadata": {"user_api_key_hash": "hash"}, "litellm_metadata": value}
)
assert metadata["user_api_key_hash"] == "hash"

View file

@ -0,0 +1,57 @@
"""The legacy Lakera guardrail reads client JSON that request setup hasn't normalized on pass-through routes."""
import json
from typing import Final
from unittest.mock import AsyncMock, patch
import httpx
import pytest
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.lakera_ai import lakeraAI_Moderation
VENDOR: Final = "http://vendor.invalid"
CLEAN: Final = httpx.Response(200, json={"results": [{"flagged": False}]}, request=httpx.Request("POST", VENDOR))
async def _sent_to_lakera(data: dict[str, object], call_type: str = "pass_through_endpoint") -> tuple[str, ...]:
guardrail: Final = lakeraAI_Moderation(api_key="key", api_base=VENDOR)
with patch.object(guardrail.async_handler, "post", new=AsyncMock(return_value=CLEAN)) as post:
await guardrail._check(data=data, user_api_key_dict=UserAPIKeyAuth(api_key="key"), call_type=call_type)
return tuple(call.kwargs["data"] for call in post.await_args_list)
@pytest.mark.asyncio
@pytest.mark.parametrize("token_arrays", [[1, 2, 3], [[1, 2], [3, 4]]])
async def test_token_arrays_have_no_text_to_send(token_arrays: list[object]) -> None:
assert await _sent_to_lakera({"input": token_arrays}) == ()
@pytest.mark.asyncio
async def test_string_entries_of_a_list_input_are_sent() -> None:
sent: Final = await _sent_to_lakera({"input": ["hello", "there"]})
assert [json.loads(body)["input"] for body in sent] == ["hello\nthere"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"content",
["card 4111", [{"type": "input_text", "text": "card 4111"}, {"type": "input_image", "image_url": "http://x"}]],
)
async def test_a_responses_message_in_a_list_input_is_sent(content: object) -> None:
sent: Final = await _sent_to_lakera({"input": [{"role": "user", "content": content}]})
assert [json.loads(body)["input"] for body in sent] == ["card 4111"]
@pytest.mark.asyncio
async def test_a_pass_through_body_with_no_messages_or_input_sends_nothing() -> None:
contents: Final = {"contents": [{"role": "user", "parts": [{"text": "hello"}]}]}
assert await _sent_to_lakera(contents) == ()
@pytest.mark.asyncio
async def test_a_native_body_with_no_messages_or_input_still_reaches_lakera() -> None:
assert await _sent_to_lakera({"prompt": "hello"}, call_type="text_completion") == ("",)

View file

@ -2413,6 +2413,35 @@ async def test_lakera_post_call_allows_clean_content():
assert result is llm_response
@pytest.mark.asyncio
async def test_post_call_with_no_messages_and_no_reply_text_returns_the_response():
guardrail: Final = LakeraAIGuardrail(api_key="test_key")
response: Final = {"candidates": [{"content": {"role": "model", "parts": [{"text": "hi"}]}}]}
rejected: Final = HTTPException(status_code=400, detail="either content, messages, or tools must be provided")
with patch.object(guardrail, "call_v2_guard", new=AsyncMock(side_effect=rejected)):
result: Final = await guardrail.async_post_call_success_hook(
data={"contents": [{"role": "user", "parts": [{"text": "hello"}]}]},
user_api_key_dict=UserAPIKeyAuth(api_key="test_key"),
response=response,
)
assert result == response
@pytest.mark.asyncio
async def test_post_call_with_messages_still_reaches_lakera():
guardrail: Final = LakeraAIGuardrail(api_key="test_key")
rejected: Final = HTTPException(status_code=400, detail="rejected")
with (
patch.object(guardrail, "call_v2_guard", new=AsyncMock(side_effect=rejected)),
pytest.raises(HTTPException),
):
await guardrail.async_post_call_success_hook(
data={"messages": [{"role": "user", "content": "hello"}]},
user_api_key_dict=UserAPIKeyAuth(api_key="test_key"),
response={},
)
@pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown")
@pytest.mark.asyncio
async def test_lakera_post_call_masks_pii_and_allows():

View file

@ -1,7 +1,9 @@
import asyncio
import os
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from litellm.proxy.guardrails.guardrail_hooks.noma import NomaV2Guardrail
@ -747,3 +749,21 @@ class TestNomaV2ApplicationIdResolution:
payload = call_mock.call_args.kwargs["payload"]
assert "application_id" not in payload
@pytest.mark.asyncio
@pytest.mark.parametrize("value", (None, 0, False, "", [], "text", 7))
async def test_noma_v2_scans_a_request_whose_litellm_metadata_is_not_an_object(value: object) -> None:
guardrail: Final = NomaV2Guardrail(api_key="key", api_base="http://vendor.invalid", guardrail_name="noma")
verdict: Final = httpx.Response(
200, json={"action": "NONE"}, request=httpx.Request("POST", "http://vendor.invalid")
)
with patch.object(guardrail.async_handler, "post", new=AsyncMock(return_value=verdict)):
result: Final = await guardrail.apply_guardrail(
inputs={"texts": ["hello"]},
request_data={"metadata": {}, "litellm_metadata": value},
input_type="request",
)
assert result["texts"] == ["hello"]

View file

@ -344,6 +344,7 @@ class TestVertexAIPassThroughHandler:
custom_headers={"Authorization": f"Bearer {test_token}"},
is_streaming_request=False,
upstream_headers_filter=mock.ANY,
inherit_auth_guardrails=True,
)
@pytest.mark.asyncio
@ -444,6 +445,7 @@ class TestVertexAIPassThroughHandler:
custom_headers={"Authorization": f"Bearer {test_token}"},
is_streaming_request=False,
upstream_headers_filter=mock.ANY,
inherit_auth_guardrails=True,
)
@pytest.mark.parametrize(
@ -540,6 +542,7 @@ class TestVertexAIPassThroughHandler:
custom_headers={"Authorization": f"Bearer {default_credentials}"},
is_streaming_request=False,
upstream_headers_filter=mock.ANY,
inherit_auth_guardrails=True,
)
@pytest.mark.asyncio

View file

@ -82,9 +82,10 @@ from litellm.proxy.pass_through_endpoints.success_handler import (
)
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.spend_tracking.spend_tracking_utils import get_spend_logs_id
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token
from litellm.proxy.utils import InternalUsageCache, ProxyLogging, _merge_guardrails_with_existing, hash_token
from litellm.types import utils as types_utils
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
from litellm.types.utils import CallTypes
from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
@ -13241,3 +13242,343 @@ async def test_passthrough_header_and_callback_share_finalized_cost(
finally:
cleanup()
await client.aclose()
class TestInheritAuthGuardrails:
"""Built-in provider routes run the caller's key, team and project guardrails on pass-through."""
@staticmethod
async def _run(
body: dict[str, object],
*,
inherit: bool,
user_api_key_dict: UserAPIKeyAuth,
guardrails: tuple[CustomGuardrail, ...] = (),
) -> tuple[list[dict[str, object]], AsyncMock]:
_, pre_call_data, post_call = await TestInheritAuthGuardrails._run_with_response(
body, inherit=inherit, user_api_key_dict=user_api_key_dict, guardrails=guardrails
)
return pre_call_data, post_call
@staticmethod
async def _run_with_response(
body: dict[str, object],
*,
inherit: bool,
user_api_key_dict: UserAPIKeyAuth,
guardrails: tuple[CustomGuardrail, ...] = (),
) -> tuple[Response, list[dict[str, object]], AsyncMock]:
pre_call_data: Final[list[dict[str, object]]] = []
async def _record_pre_call(**kwargs: object) -> object:
pre_call_data.append(json.loads(json.dumps(kwargs["data"], default=str)))
return kwargs["data"]
proxy_logging: Final = MagicMock()
proxy_logging.pre_call_hook = AsyncMock(side_effect=_record_pre_call)
proxy_logging.post_call_success_hook = AsyncMock(return_value={"id": "chatcmpl-1"})
proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={})
upstream: Final = httpx.Response(
status_code=200,
headers={"content-type": "application/json"},
content=b'{\n "id":"chatcmpl-1"\n}',
request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),
)
request: Final = MagicMock()
request.method = "POST"
request.query_params = {}
request.headers = MagicMock()
request.headers.copy.return_value = {}
module: Final = "litellm.proxy.pass_through_endpoints.pass_through_endpoints"
with ExitStack() as stack:
stack.enter_context(
patch(
f"{module}.HttpPassThroughEndpointHelpers.non_streaming_http_request_handler",
new_callable=AsyncMock,
return_value=upstream,
)
)
stack.enter_context(patch(f"{module}._is_streaming_response", return_value=False))
stack.enter_context(patch("litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging))
stack.enter_context(patch("litellm.proxy.proxy_server.premium_user", True))
stack.enter_context(
patch(
f"{module}.pass_through_endpoint_logging",
MagicMock(pass_through_async_success_handler=AsyncMock()),
)
)
stack.enter_context(patch(f"{module}.read_request_body", new_callable=AsyncMock, return_value=body))
stack.enter_context(patch(f"{module}.safe_get_request_headers", return_value={}))
stack.enter_context(patch.object(litellm, "callbacks", list(guardrails)))
response: Final = await pass_through_request(
request=request,
target="https://api.openai.com/v1/chat/completions",
custom_headers={},
user_api_key_dict=user_api_key_dict,
stream=False,
inherit_auth_guardrails=inherit,
)
return response, pre_call_data, proxy_logging.post_call_success_hook
@staticmethod
def _guardrail(name: str, event_hook: GuardrailEventHooks, *, default_on: bool = False) -> CustomGuardrail:
return CustomGuardrail(guardrail_name=name, event_hook=event_hook, default_on=default_on)
@staticmethod
def _chat_body(**extra: object) -> dict[str, object]:
return {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}], **extra}
@pytest.mark.asyncio
async def test_key_team_and_project_guardrails_reach_pre_and_post_call(self):
auth: Final = UserAPIKeyAuth(
api_key="sk-test",
metadata={"guardrails": ["key-guard"]},
team_metadata={"guardrails": ["team-guard"]},
project_metadata={"guardrails": ["project-guard"]},
)
pre_call, post_call = await self._run(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(
self._guardrail("key-guard", GuardrailEventHooks.pre_call),
self._guardrail("team-guard", GuardrailEventHooks.post_call),
),
)
assert sorted(pre_call[0]["metadata"]["guardrails"]) == ["key-guard", "project-guard", "team-guard"]
post_call.assert_awaited_once()
assert sorted(post_call.call_args.kwargs["data"]["metadata"]["guardrails"]) == [
"key-guard",
"project-guard",
"team-guard",
]
@pytest.mark.asyncio
async def test_post_call_guardrails_stay_named_after_a_model_guardrail_merge(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
_, post_call = await self._run(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("key-guard", GuardrailEventHooks.post_call),),
)
merged: Final = _merge_guardrails_with_existing(post_call.call_args.kwargs["data"], ["model-guard"])
assert merged["metadata"]["guardrails"] == ["key-guard", "model-guard"]
@pytest.mark.asyncio
@pytest.mark.parametrize("settings", [{}, True])
async def test_guardrail_a_client_names_in_the_dict_form_stays_after_the_keys(self, settings: object):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, _ = await self._run(
{**self._chat_body(), "metadata": {"guardrails": {"client-guard": settings}}},
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("key-guard", GuardrailEventHooks.pre_call),),
)
assert pre_call[0]["metadata"]["guardrails"] == ["key-guard", "client-guard"]
@staticmethod
def _body_with_policy_pipeline(mode: str) -> dict[str, object]:
pipeline: Final = GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="step-guard")])
return {
**TestInheritAuthGuardrails._chat_body(),
"metadata": {
"_guardrail_pipelines": [("policy", pipeline)],
"_pipeline_managed_guardrails": {"step-guard"},
},
}
@pytest.mark.asyncio
async def test_post_call_policy_pipeline_starts_the_post_call_pass_with_its_state(self):
_, post_call = await self._run(
self._body_with_policy_pipeline("post_call"),
inherit=True,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
)
post_call.assert_awaited_once()
metadata: Final = post_call.await_args.kwargs["data"]["metadata"]
assert metadata["_pipeline_managed_guardrails"] == {"step-guard"}
assert [policy for policy, _ in metadata["_guardrail_pipelines"]] == ["policy"]
@pytest.mark.asyncio
@pytest.mark.parametrize(("mode", "inherit"), [("pre_call", True), ("post_call", False)])
async def test_policy_pipeline_leaves_the_response_alone_without_a_post_call_run(self, mode: str, inherit: bool):
response, _, post_call = await self._run_with_response(
self._body_with_policy_pipeline(mode),
inherit=inherit,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
)
post_call.assert_not_awaited()
assert response.body == b'{\n "id":"chatcmpl-1"\n}'
@pytest.mark.asyncio
async def test_dict_form_stays_as_sent_for_a_key_without_guardrails(self):
pre_call, post_call = await self._run(
{**self._chat_body(), "metadata": {"guardrails": {"client-guard": {}}}},
inherit=True,
user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
guardrails=(self._guardrail("client-guard", GuardrailEventHooks.post_call),),
)
assert pre_call[0]["metadata"]["guardrails"] == {"client-guard": {}}
post_call.assert_not_awaited()
@pytest.mark.asyncio
async def test_post_call_pass_reads_opt_outs_from_the_key_and_team_not_the_body(self):
auth: Final = UserAPIKeyAuth(
api_key="sk-test",
metadata={"guardrails": ["key-guard"], "disable_global_guardrails": True},
team_metadata={"opted_out_global_guardrails": ["global-post"]},
)
forged: Final = {"user_api_key_metadata": {"disable_global_guardrails": False}}
_, post_call = await self._run(
{**self._chat_body(), "metadata": forged},
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("key-guard", GuardrailEventHooks.post_call),),
)
metadata: Final = post_call.call_args.kwargs["data"]["metadata"]
assert metadata["user_api_key_metadata"]["disable_global_guardrails"] is True
assert metadata["user_api_key_team_metadata"] == {"opted_out_global_guardrails": ["global-post"]}
@pytest.mark.asyncio
async def test_inherited_post_call_pass_runs_only_the_guardrails_the_request_selected(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
_, post_call = await self._run(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("key-guard", GuardrailEventHooks.post_call),),
)
assert post_call.call_args.kwargs["guardrails_only"] is True
@pytest.mark.asyncio
async def test_named_global_guardrail_the_key_opted_out_of_does_not_start_the_post_call_pass(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"opted_out_global_guardrails": ["global-post"]})
response, _, post_call = await self._run_with_response(
{**self._chat_body(), "metadata": {"guardrails": ["global-post"]}},
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("global-post", GuardrailEventHooks.post_call, default_on=True),),
)
post_call.assert_not_awaited()
assert response.body == b'{\n "id":"chatcmpl-1"\n}'
@pytest.mark.asyncio
async def test_pre_call_only_guardrails_relay_the_response_untouched(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
response, pre_call, post_call = await self._run_with_response(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(
self._guardrail("key-guard", GuardrailEventHooks.pre_call),
self._guardrail("global-post", GuardrailEventHooks.post_call, default_on=True),
),
)
assert pre_call[0]["metadata"]["guardrails"] == ["key-guard"]
post_call.assert_not_awaited()
assert response.body == b'{\n "id":"chatcmpl-1"\n}'
@pytest.mark.asyncio
async def test_a_named_default_on_post_call_guardrail_still_checks_the_response(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["global-post"]})
_, post_call = await self._run(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("global-post", GuardrailEventHooks.post_call, default_on=True),),
)
post_call.assert_awaited_once()
assert post_call.call_args.kwargs["data"]["metadata"]["guardrails"] == ["global-post"]
@pytest.mark.asyncio
async def test_post_call_guardrails_get_the_request_logging_object(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-post"]})
_, post_call = await self._run(
self._chat_body(),
inherit=True,
user_api_key_dict=auth,
guardrails=(self._guardrail("key-post", GuardrailEventHooks.post_call),),
)
post_call.assert_awaited_once()
assert isinstance(post_call.call_args.kwargs["data"]["litellm_logging_obj"], LiteLLMLoggingObj)
@pytest.mark.asyncio
async def test_request_body_guardrails_are_added_to_the_key_guardrails(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, _ = await self._run(
self._chat_body(guardrails=["body-guard"]), inherit=True, user_api_key_dict=auth
)
assert pre_call[0]["metadata"]["guardrails"] == ["key-guard", "body-guard"]
assert "guardrails" not in pre_call[0]
@pytest.mark.asyncio
async def test_guardrails_the_client_names_in_metadata_still_run_for_a_guarded_key(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, post_call = await self._run(
self._chat_body(metadata={"guardrails": ["client-pre", "client-post"]}),
inherit=True,
user_api_key_dict=auth,
guardrails=(
self._guardrail("key-guard", GuardrailEventHooks.pre_call),
self._guardrail("client-post", GuardrailEventHooks.post_call),
),
)
assert pre_call[0]["metadata"] == {"guardrails": ["key-guard", "client-pre", "client-post"]}
post_call.assert_awaited_once()
assert post_call.call_args.kwargs["data"]["metadata"]["guardrails"] == [
"key-guard",
"client-pre",
"client-post",
]
@pytest.mark.asyncio
async def test_empty_request_body_guardrails_keep_the_key_guardrails(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, _ = await self._run(self._chat_body(guardrails=[]), inherit=True, user_api_key_dict=auth)
assert pre_call[0]["metadata"]["guardrails"] == ["key-guard"]
@pytest.mark.asyncio
async def test_routes_without_inheritance_keep_key_guardrails_off(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, post_call = await self._run(self._chat_body(), inherit=False, user_api_key_dict=auth)
assert "guardrails" not in pre_call[0].get("metadata", {})
post_call.assert_not_awaited()
@pytest.mark.asyncio
async def test_request_without_a_body_gets_no_guardrails(self):
auth: Final = UserAPIKeyAuth(api_key="sk-test", metadata={"guardrails": ["key-guard"]})
pre_call, post_call = await self._run({}, inherit=True, user_api_key_dict=auth)
assert "guardrails" not in pre_call[0].get("metadata", {})
post_call.assert_not_awaited()

View file

@ -18,6 +18,7 @@ from litellm.integrations.custom_guardrail import (
ModifyResponseException,
)
from litellm.proxy._types import ProxyException
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import _run_post_call_guardrails_keeping_the_charge
_PT_MOD = "litellm.proxy.pass_through_endpoints.pass_through_endpoints"
_COLLECT = "litellm.proxy.pass_through_endpoints.passthrough_guardrails.PassthroughGuardrailHandler.collect_guardrails"
@ -130,6 +131,29 @@ class TestPassthroughPostCallGuardrails:
call_kwargs = mock_proxy_logging.post_call_success_hook.call_args
assert call_kwargs.kwargs["response"] == _GEMINI_RESPONSE
@patch(_COLLECT, return_value=["rubrik"])
async def test_configured_endpoint_keeps_every_post_call_hook_and_reads_no_key_opt_outs(
self,
mock_collect,
):
mock_proxy_logging = MagicMock()
mock_proxy_logging.pre_call_hook = AsyncMock(return_value={})
mock_proxy_logging.post_call_success_hook = AsyncMock(return_value=_GEMINI_RESPONSE)
mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={})
with _common_patches(mock_proxy_logging, _make_httpx_response(_GEMINI_RESPONSE)):
await pass_through_request(
request=_make_mock_request(),
target="https://example.com/v1/generateContent",
custom_headers={"Content-Type": "application/json"},
user_api_key_dict=_make_user_api_key_dict(),
stream=False,
)
call_kwargs = mock_proxy_logging.post_call_success_hook.call_args.kwargs
assert call_kwargs["guardrails_only"] is False
assert call_kwargs["data"]["metadata"].keys() == {"guardrails"}
@patch(_COLLECT, return_value=[])
async def test_post_call_success_hook_skipped_when_no_guardrails(
self,
@ -216,7 +240,7 @@ class TestPassthroughPostCallGuardrails:
block path dropping it."""
mock_response = _make_httpx_response(_GEMINI_RESPONSE)
def _block(*, data, user_api_key_dict, response):
def _block(*, data, user_api_key_dict, response, guardrails_only):
metadata = data.setdefault("metadata", {})
metadata.setdefault("standard_logging_guardrail_information", []).append(
{"guardrail_name": "rubrik", "guardrail_status": "guardrail_intervened"}
@ -312,3 +336,37 @@ def test_modify_response_exception_importable_from_both_paths():
)
assert FromExceptions is FromGuardrail
@pytest.mark.asyncio
async def test_a_blocked_response_settles_the_provider_charge_and_keeps_the_block():
blocked = HTTPException(status_code=400, detail="blocked")
settle = AsyncMock()
with pytest.raises(HTTPException) as caught:
await _run_post_call_guardrails_keeping_the_charge(AsyncMock(side_effect=blocked), settle)
assert caught.value is blocked
settle.assert_awaited_once()
@pytest.mark.asyncio
async def test_a_failed_settlement_does_not_replace_the_block():
blocked = HTTPException(status_code=400, detail="blocked")
with pytest.raises(HTTPException) as caught:
await _run_post_call_guardrails_keeping_the_charge(
AsyncMock(side_effect=blocked), AsyncMock(side_effect=ValueError("no price"))
)
assert caught.value is blocked
@pytest.mark.asyncio
async def test_an_allowed_response_leaves_the_charge_to_the_success_path():
settle = AsyncMock()
response = await _run_post_call_guardrails_keeping_the_charge(AsyncMock(return_value={"id": "kept"}), settle)
assert response == {"id": "kept"}
settle.assert_not_awaited()

View file

@ -5327,6 +5327,33 @@ async def test_embedding_header_forwarding_without_model_group_config():
litellm.model_group_settings = original_model_group_settings
@pytest.mark.asyncio
@pytest.mark.parametrize("model", [0, False, [], {}, 7, True])
async def test_policy_engine_leaves_a_model_that_is_not_a_string_alone(
model: object, monkeypatch: pytest.MonkeyPatch
) -> None:
from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
from litellm.types.proxy.policy_engine import Policy, PolicyAttachment, PolicyGuardrails
policy_registry = get_policy_registry()
monkeypatch.setattr(
policy_registry, "_policies", {"baseline": Policy(guardrails=PolicyGuardrails(add=["pii_blocker"]))}
)
monkeypatch.setattr(policy_registry, "_initialized", True)
attachment_registry = get_attachment_registry()
monkeypatch.setattr(attachment_registry, "_attachments", [PolicyAttachment(policy="baseline", scope="*")])
monkeypatch.setattr(attachment_registry, "_initialized", True)
data = {"model": model, "metadata": {}}
await add_guardrails_from_policy_engine(
data=data, metadata_variable_name="metadata", user_api_key_dict=UserAPIKeyAuth(api_key="test-key")
)
assert data["model"] == model
assert data["metadata"]["guardrails"] == ["pii_blocker"]
@pytest.mark.asyncio
async def test_add_guardrails_from_policy_engine():
"""

View file

@ -275,54 +275,45 @@ async def test_process_guardrail_callback_enriches_and_reraises_http_exception(
# ---------------------------------------------------------------------------
def test_process_guardrail_metadata_calls_header_helper(proxy_logging, monkeypatch):
calls: List[Dict[str, Any]] = []
def fake_add(request_data, guardrail_name):
calls.append({"data": request_data, "name": guardrail_name})
from litellm.proxy.common_utils import callback_utils
monkeypatch.setattr(callback_utils, "add_guardrail_to_applied_guardrails_header", fake_add)
data = {"metadata": {"guardrails": ["g1", "g2"]}}
def test_process_guardrail_metadata_adds_each_guardrail_once_in_order(proxy_logging):
data = {"metadata": {"guardrails": ["g1", "g2", "g1"]}}
proxy_logging._process_guardrail_metadata(data)
snapshot = {
"call_count": len(calls),
"first_name": calls[0]["name"],
"second_name": calls[1]["name"],
"data_passed_is_input": all(c["data"] is data for c in calls),
}
assert snapshot == {
"call_count": 2,
"first_name": "g1",
"second_name": "g2",
"data_passed_is_input": True,
}
assert data["metadata"]["applied_guardrails"] == ["g1", "g2"]
def test_process_guardrail_metadata_skips_already_applied(proxy_logging, monkeypatch):
calls: List[str] = []
def fake_add(request_data, guardrail_name):
calls.append(guardrail_name)
from litellm.proxy.common_utils import callback_utils
monkeypatch.setattr(callback_utils, "add_guardrail_to_applied_guardrails_header", fake_add)
def test_process_guardrail_metadata_skips_already_applied(proxy_logging):
data = {"metadata": {"guardrails": ["g1", "g2"], "applied_guardrails": ["g1"]}}
proxy_logging._process_guardrail_metadata(data)
assert calls == ["g2"]
assert data["metadata"]["applied_guardrails"] == ["g1", "g2"]
def test_process_guardrail_metadata_no_metadata_is_noop(proxy_logging, monkeypatch):
from litellm.proxy.common_utils import callback_utils
def test_process_guardrail_metadata_records_in_the_litellm_metadata_bucket(proxy_logging):
data = {"litellm_metadata": {"guardrails": ["g1", "g2"], "applied_guardrails": ["g2"]}}
proxy_logging._process_guardrail_metadata(data)
assert data == {"litellm_metadata": {"guardrails": ["g1", "g2"], "applied_guardrails": ["g2", "g1"]}}
monkeypatch.setattr(
callback_utils,
"add_guardrail_to_applied_guardrails_header",
MagicMock(side_effect=AssertionError("should not be called")),
)
proxy_logging._process_guardrail_metadata({})
def test_process_guardrail_metadata_does_not_search_the_applied_list_per_name(proxy_logging):
class _NoMembershipChecks(list):
def __contains__(self, item):
raise AssertionError("searched the applied list")
names = [f"guard-{i}" for i in range(100)]
data = {"metadata": {"guardrails": names, "applied_guardrails": _NoMembershipChecks()}}
proxy_logging._process_guardrail_metadata(data)
assert data["metadata"]["applied_guardrails"] == names
def test_process_guardrail_metadata_no_metadata_is_noop(proxy_logging):
data: Dict[str, Any] = {}
proxy_logging._process_guardrail_metadata(data)
assert data == {}
def test_process_guardrail_metadata_with_only_settings_entries_leaves_data_unchanged(proxy_logging):
data = {"metadata": {"guardrails": [{"g1": {"extra_body": {}}}]}}
proxy_logging._process_guardrail_metadata(data)
assert data == {"metadata": {"guardrails": [{"g1": {"extra_body": {}}}]}}
def test_process_guardrail_metadata_invalid_data_raises(proxy_logging):

View file

@ -59,6 +59,47 @@ async def test_post_call_success_hook_runs_other_callback_and_replaces_response(
assert out == new_response
@pytest.mark.asyncio
@pytest.mark.parametrize(
("guardrails_only", "guardrails_seen"), [(False, ["key-guard", "model-guard"]), (True, ["key-guard"])]
)
async def test_post_call_success_hook_merges_model_level_guardrails_unless_guardrails_only(
proxy_logging, make_user_api_key_auth, monkeypatch, guardrails_only, guardrails_seen
):
router = MagicMock()
router.registry.return_value.deployments.return_value = [{"litellm_params": {"guardrails": ["model-guard"]}}]
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", router)
g = _make_guardrail(should_run=False)
monkeypatch.setattr(litellm, "callbacks", [g])
await proxy_logging.post_call_success_hook(
data={"model": "alias", "metadata": {"guardrails": ["key-guard"]}},
response=MagicMock(),
user_api_key_dict=make_user_api_key_auth(),
guardrails_only=guardrails_only,
)
assert g.should_run_guardrail.call_args.kwargs["data"]["metadata"]["guardrails"] == guardrails_seen
@pytest.mark.asyncio
async def test_post_call_success_hook_guardrails_only_leaves_other_callbacks_out(
proxy_logging, make_user_api_key_auth, monkeypatch
):
other = MagicMock(spec=CustomLogger)
other.async_post_call_success_hook = AsyncMock(return_value={"replaced": True})
g = _make_guardrail(override={"guarded": True})
monkeypatch.setattr(litellm, "callbacks", [other, g])
out = await proxy_logging.post_call_success_hook(
data={}, response={"original": True}, user_api_key_dict=make_user_api_key_auth(), guardrails_only=True
)
other.async_post_call_success_hook.assert_not_called()
g.async_post_call_success_hook.assert_awaited_once()
assert out == {"guarded": True}
@pytest.mark.asyncio
async def test_post_call_success_hook_guardrail_should_not_run_skipped(
proxy_logging, make_user_api_key_auth, monkeypatch