From ee62b850597c30d908093835483a6ba7bbb82a1d Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Mon, 5 Oct 2026 08:08:41 -0500 Subject: [PATCH 1/9] feat(guardrails): add the ThirdLaw guardrail integration Adds a `thirdlaw` guardrail built on native lifecycle hooks. It posts the full request and response bodies, headers and provider response headers to ThirdLaw's /guardrails/litellm/v2 endpoint and applies allow, block, modify_request and modify_response decisions, including on chat, Responses and Messages streams scanned in their own body shape. Adds the ThirdLaw card and logo to the Admin UI guardrail garden --- litellm/proxy/guardrails/anthropic_sse.py | 180 +- .../guardrail_hooks/thirdlaw/__init__.py | 44 + .../guardrail_hooks/thirdlaw/thirdlaw.py | 1338 ++++++++++ litellm/proxy/guardrails/stream_surface.py | 149 ++ litellm/types/guardrails.py | 5 + .../guardrails/guardrail_hooks/thirdlaw.py | 130 + tests/documentation_tests/test_env_keys.py | 14 +- .../guardrail_hooks/test_thirdlaw.py | 2268 +++++++++++++++++ .../proxy/guardrails/test_anthropic_sse.py | 106 + .../proxy/guardrails/test_stream_surface.py | 122 + .../public/assets/logos/thirdlaw.svg | 84 + .../_components/guardrail_garden_configs.ts | 6 + .../_components/guardrail_garden_data.test.ts | 1 + .../_components/guardrail_garden_data.ts | 10 + .../guardrail_info_helpers.test.tsx | 14 + .../_components/guardrail_info_helpers.tsx | 3 + 16 files changed, 4472 insertions(+), 2 deletions(-) create mode 100644 litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py create mode 100644 litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py create mode 100644 litellm/proxy/guardrails/stream_surface.py create mode 100644 litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py create mode 100644 tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py create mode 100644 tests/unit/proxy/guardrails/test_anthropic_sse.py create mode 100644 tests/unit/proxy/guardrails/test_stream_surface.py create mode 100644 ui/litellm-dashboard/public/assets/logos/thirdlaw.svg diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index c446cfaa815..1586ec88b9d 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -10,10 +10,24 @@ from __future__ import annotations import codecs import json from collections.abc import Mapping, Sequence -from typing import Final +from dataclasses import dataclass, replace +from functools import reduce +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + cast, # noqa: TID251 # the Messages body is a total=False TypedDict; validating through it drops provider fields +) from litellm.types.utils import Choices, ModelResponse +if TYPE_CHECKING: + from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse + +_EMPTY_MAP: Final[Mapping[str, object]] = MappingProxyType({}) + +_TOOL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"}) + _ANTHROPIC_EVENT_TYPES: Final = frozenset( { "message_start", @@ -32,6 +46,11 @@ def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool: return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks) +def sse_stream_text(all_chunks: Sequence[object]) -> str | None: + """The raw SSE frames joined as text, for a caller that forwards the stream instead of folding it.""" + return _joined_sse_stream(all_chunks) + + def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None: raw: Final = b"".join( chunk if isinstance(chunk, bytes) else chunk.encode("utf-8") @@ -178,3 +197,162 @@ def anthropic_sse_chunks_from_response(assembled: ModelResponse) -> tuple[bytes, response=assembled ) return tuple(FakeAnthropicMessagesStreamIterator(response=anthropic_response).chunks) + + +def _str_field(source: Mapping[str, object], key: str) -> str: + return value if isinstance(value := source.get(key), str) else "" + + +def _accumulated_tool_input(partial_json: Sequence[str]) -> object: + """Assemble a tool block's streamed arguments. + + A stream cut mid-block leaves unparseable JSON. Returning an empty input there would + drop whatever the model had already emitted, and a caller can force exactly that by + prompting for restricted content inside a tool argument and letting max_tokens truncate + it: the text would reach the client in the replayed frames having never been scanned. + So the raw fragment is carried under ``_raw`` instead, keeping the block a well-formed + object while leaving the text visible to whatever inspects the assembled body. + """ + joined: Final = "".join(partial_json) + if not joined: + return {} + try: + return json.loads(joined) + except json.JSONDecodeError: + return {"_raw": joined} + + +@dataclass(frozen=True, slots=True) +class _ContentBlock: + """A ``content_block_start`` and every delta that has landed on it since.""" + + start: Mapping[str, object] + text: tuple[str, ...] = () + thinking: tuple[str, ...] = () + signature: tuple[str, ...] = () + partial_json: tuple[str, ...] = () + citations: tuple[object, ...] = () + + def with_delta(self, delta: Mapping[str, object]) -> _ContentBlock: + match delta.get("type"): + case "text_delta": + return replace(self, text=(*self.text, _str_field(delta, "text"))) + case "thinking_delta": + return replace(self, thinking=(*self.thinking, _str_field(delta, "thinking"))) + case "signature_delta": + return replace(self, signature=(*self.signature, _str_field(delta, "signature"))) + case "input_json_delta": + return replace(self, partial_json=(*self.partial_json, _str_field(delta, "partial_json"))) + case "citations_delta": + citation: Final = delta.get("citation") + return self if citation is None else replace(self, citations=(*self.citations, citation)) + case _: + return self + + def _accumulated_fields(self) -> Mapping[str, object]: + block_type: Final = self.start.get("type") + if block_type == "text": + return {"text": "".join(self.text)} + if block_type == "thinking": + return {"thinking": "".join(self.thinking), "signature": "".join(self.signature)} + if block_type in _TOOL_BLOCK_TYPES: + return {"input": _accumulated_tool_input(self.partial_json)} + return _EMPTY_MAP + + def rendered(self) -> dict[str, object]: + cited: Final[Mapping[str, object]] = ( + MappingProxyType({"citations": tuple(self.citations)}) if self.citations else _EMPTY_MAP + ) + return {**self.start, **self._accumulated_fields(), **cited} + + +_EMPTY_BLOCKS: Final[Mapping[int, _ContentBlock]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class _AnthropicMessage: + envelope: Mapping[str, object] | None = None + blocks: Mapping[int, _ContentBlock] = _EMPTY_BLOCKS + stop_reason: object = None + stop_sequence: object = None + delta_usage: Mapping[str, object] = _EMPTY_MAP + + +def _with_block(state: _AnthropicMessage, index: int, block: _ContentBlock) -> _AnthropicMessage: + return replace(state, blocks=MappingProxyType({**state.blocks, index: block})) + + +def _with_message_delta(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage: + delta: Final = event.get("delta") + usage: Final = event.get("usage") + stops: Final = delta if isinstance(delta, Mapping) else _EMPTY_MAP + return replace( + state, + stop_reason=stops.get("stop_reason", state.stop_reason), + stop_sequence=stops.get("stop_sequence", state.stop_sequence), + delta_usage=( + MappingProxyType({**state.delta_usage, **usage}) if isinstance(usage, Mapping) else state.delta_usage + ), + ) + + +def _with_event(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage: + match event.get("type"): + case "message_start": + message: Final = event.get("message") + return replace(state, envelope=message) if isinstance(message, Mapping) else state + case "content_block_start": + index: Final = event.get("index") + block: Final = event.get("content_block") + if not isinstance(index, int) or not isinstance(block, Mapping): + return state + return _with_block(state, index, _ContentBlock(start=block)) + case "content_block_delta": + delta_index: Final = event.get("index") + delta: Final = event.get("delta") + if not isinstance(delta_index, int) or not isinstance(delta, Mapping): + return state + started: Final = state.blocks.get(delta_index) + return state if started is None else _with_block(state, delta_index, started.with_delta(delta)) + case "message_delta": + return _with_message_delta(state, event) + case _: + return state + + +def assemble_anthropic_sse_body(all_chunks: Sequence[object]) -> Mapping[str, object] | None: + """Fold raw Anthropic SSE frames back into the non-streaming Messages body. + + ``assemble_anthropic_sse_stream`` folds them into a ``ModelResponse``, which is all a guardrail + scanning text needs. A guardrail that posts the provider body itself needs the shape the + non-streaming route already sends, because the chat-completions one has no field for thinking + blocks, citations, ``stop_sequence`` or the cache token split and drops them silently. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return None + state: Final = reduce(_with_event, _parsed_sse_events(sse_stream), _AnthropicMessage()) + envelope: Final = state.envelope + if envelope is None: + return None + started_usage: Final = envelope.get("usage") + return { + **envelope, + "content": tuple(block.rendered() for _, block in sorted(state.blocks.items())), + "stop_reason": state.stop_reason if state.stop_reason is not None else envelope.get("stop_reason"), + "stop_sequence": state.stop_sequence if state.stop_sequence is not None else envelope.get("stop_sequence"), + "usage": { + **(started_usage if isinstance(started_usage, Mapping) else _EMPTY_MAP), + **state.delta_usage, + }, + } + + +def anthropic_sse_chunks_from_body(body: Mapping[str, object]) -> tuple[bytes, ...]: + """Re-emit a native Anthropic Messages body as the SSE frames its client expects.""" + from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, + ) + + typed: Final = cast("AnthropicMessagesResponse", dict(body)) + return tuple(FakeAnthropicMessagesStreamIterator(response=typed).chunks) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py new file mode 100644 index 00000000000..5ae2f3bee03 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py @@ -0,0 +1,44 @@ +from typing import TYPE_CHECKING, Final + +from litellm.types.guardrails import SupportedGuardrailIntegrations + +from .thirdlaw import ThirdlawGuardrail + +if TYPE_CHECKING: + from litellm.types.guardrails import Guardrail, LitellmParams + + +def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> ThirdlawGuardrail: + import litellm + + # LitellmParams' multiple-inheritance MRO resolves guardrail_timeout to a + # None default, so the 60s fallback must be applied here. + _thirdlaw_callback: Final = ThirdlawGuardrail( + api_base=litellm_params.api_base, + api_key=litellm_params.api_key, + additional_headers=getattr(litellm_params, "additional_headers", None), + guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None) or 60, + streaming_buffer_until_moderated=getattr(litellm_params, "streaming_buffer_until_moderated", True), + streaming_end_of_stream_only=getattr(litellm_params, "streaming_end_of_stream_only", True), + streaming_sampling_rate=getattr(litellm_params, "streaming_sampling_rate", 5), + unreachable_fallback=getattr(litellm_params, "unreachable_fallback", None) or "fail_closed", + unscannable_stream_fallback=getattr(litellm_params, "unscannable_stream_fallback", None) or "fail_closed", + send_stream_chunks=getattr(litellm_params, "send_stream_chunks", None) is True, + additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", None), + headers=getattr(litellm_params, "headers", None), + extra_headers=getattr(litellm_params, "extra_headers", None), + guardrail_name=guardrail.get("guardrail_name", ""), + event_hook=litellm_params.mode, + default_on=litellm_params.default_on, + ) + litellm.logging_callback_manager.add_litellm_callback(_thirdlaw_callback) + return _thirdlaw_callback + + +guardrail_initializer_registry: Final = { # mutable-ok: the registry loader requires a plain dict + SupportedGuardrailIntegrations.THIRDLAW.value: initialize_guardrail, +} + +guardrail_class_registry: Final = { # mutable-ok: the registry loader requires a plain dict + SupportedGuardrailIntegrations.THIRDLAW.value: ThirdlawGuardrail, +} diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py new file mode 100644 index 00000000000..6761859b509 --- /dev/null +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -0,0 +1,1338 @@ +import json +import time +from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence +from datetime import datetime, timezone +from types import MappingProxyType +from typing import ( + TYPE_CHECKING, + Final, + Literal, + TypeAlias, + cast, # noqa: TID251 # SSE byte frames ride the ModelResponseStream-typed pipe (bedrock precedent) +) + +import httpx +from pydantic import BaseModel, TypeAdapter, ValidationError + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm._version import version as litellm_version +from litellm.caching import DualCache +from litellm.exceptions import GuardrailRaisedException +from litellm.exceptions import Timeout as LitellmTimeout +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + log_guardrail_information, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + httpxSpecialProvider, +) +from litellm.proxy._types import ( + LiteLLM_ManagementEndpoint_MetadataFields, + LiteLLM_ManagementEndpoint_MetadataFields_Premium, + UserAPIKeyAuth, +) +from litellm.proxy.guardrails.anthropic_sse import ( + anthropic_sse_chunks_from_body, + anthropic_sse_error_frames, + assemble_anthropic_sse_body, + sse_stream_text, +) +from litellm.proxy.guardrails.stream_surface import ( + StreamSurface, + classify_stream, + final_responses_api_response, + is_terminal_error_stream, + responses_deltas_absent_from_body, +) +from litellm.proxy.litellm_pre_call_utils import ( + _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # shared list + _UNTRUSTED_ROOT_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # shared list +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrailRequest, + ThirdlawGuardrailRequestMetadata, + ThirdlawGuardrailResponse, +) +from litellm.types.utils import ( + CallTypesLiteral, + GuardrailStatus, + LLMResponseTypes, + ModelResponse, + ModelResponseStream, +) + +if TYPE_CHECKING: + from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel + +GUARDRAIL_NAME: Final = "thirdlaw" + +_ENDPOINT_PATH: Final = "/guardrails/litellm/v2" + +_UNREACHABLE_STATUS_CODES: Final = frozenset({502, 503, 504}) + +# Credential-bearing LiteLLM parameters that are not declared on ``CredentialLiteLLMParams`` +# because callers supply them through **kwargs. ``extra_headers`` / ``default_headers`` are +# here because either can carry a provider Authorization header. +_KWARGS_ONLY_CREDENTIAL_KEYS: Final = frozenset( + { + "azure_ad_token_provider", + "default_headers", + "extra_headers", + "oci_fingerprint", + "oci_key", + "oci_key_file", + "oci_signer", + "oci_tenancy", + "oci_user", + } +) + + +def _credential_keys() -> frozenset[str]: + """Every LiteLLM parameter that can carry a provider credential. + + Derived from ``CredentialLiteLLMParams`` rather than enumerated here, so a provider + credential field added upstream is withheld from ThirdLaw without a matching edit in + this module. The model also carries non-secret routing fields (``api_base``, + ``vertex_location``, ``aws_region_name``); those are not provider body content either, + so stripping the whole model costs nothing. + """ + from litellm.types.router import CredentialLiteLLMParams + + return frozenset(CredentialLiteLLMParams.model_fields) | _KWARGS_ONLY_CREDENTIAL_KEYS + + +# Not part of the provider request body. ``secret_fields`` holds plaintext Authorization +# values, and every credential-bearing litellm parameter is withheld alongside it: a caller +# authenticating with aws_secret_access_key, azure_ad_token, client_secret or +# vertex_credentials must not have that value serialized out to the guardrail service. +_BODY_STRIP_KEYS: Final = _credential_keys() | frozenset( + { + "guardrail_config", + "guardrails", + "headers", + "litellm_call_id", + "litellm_logging_obj", + "litellm_metadata", + "litellm_session_id", + "litellm_trace_id", + "metadata", + "provider_specific_header", + "proxy_server_request", + "response", + "responses", + "secret_fields", + "user_api_key_dict", + } +) + +# Deployment selectors: either one redirects the call to different credentials or a different +# provider than the key/team was authorized against. +_ROUTING_SELECTOR_KEYS: Final = frozenset({"custom_llm_provider", "litellm_credential_name"}) + +# Never writable by modify_request: ``guardrails`` gates which guardrails run, ``model`` was +# authorized against the key/team before this hook, ``stream`` changes the wire protocol mid-request. +# ``_UNTRUSTED_ROOT_CONTROL_FIELDS`` is litellm's own list of root controls stripped from caller +# input by add_litellm_data_to_request; a guardrail response arrives after that strip, so without +# this it could put back agentic-loop and sandbox-interception state the caller was denied. +_WRITE_BACK_DENY_KEYS: Final = ( + _BODY_STRIP_KEYS + | _ROUTING_SELECTOR_KEYS + | frozenset(_UNTRUSTED_ROOT_CONTROL_FIELDS) + | frozenset({"model", "policies", "stream", "user"}) +) + +_USER_METADATA_FIELDS: Final = ( + "user_api_key_hash", + "user_api_key_alias", + "user_api_key_user_id", + "user_api_key_user_email", + "user_api_key_team_id", + "user_api_key_team_alias", + "user_api_key_end_user_id", + "user_api_key_org_id", + "user_api_key_project_id", + "user_api_key_project_alias", + "user_api_key_org_alias", + # Unprefixed unlike its siblings above: litellm writes this key bare, not as + # "user_api_key_agent_id" (see add_user_api_key_auth_to_request_metadata). + "agent_id", +) + +# Admin-facing control keys litellm reserves inside a key/team/project/org's own custom +# `metadata` dict -- never admin-typed custom data. Combines the management-endpoint +# reserved fields (rate/budget overrides, guardrails, tags, etc.) with litellm's own +# client-forgery denylist (guardrail-bypass and internal routing/logging signals): the +# same reasoning that makes these untrustworthy coming from a caller makes them +# unsuitable to forward to a third-party guardrail. ``model_config`` is added +# separately -- it can carry ``litellm_credentials`` selector names for provider +# routing, which is proxy operational config, not admin-authored metadata. +_RESERVED_METADATA_KEYS: Final = frozenset( + ( + *LiteLLM_ManagementEndpoint_MetadataFields, + *LiteLLM_ManagementEndpoint_MetadataFields_Premium, + *_UNTRUSTED_METADATA_CONTROL_FIELDS, + "model_config", + ) +) + +_WireEvent: TypeAlias = Literal["pre_call", "during_call", "post_call"] + +# What a buffered stream assembles into, one shape per surface that has an assembler +_AssembledStream: TypeAlias = ModelResponse | ResponsesAPIResponse | Mapping[str, object] + +# Never writable by modify_response on a /v1/responses stream: litellm encrypts the response id and +# the client chains the next turn off it with previous_response_id. +_RESPONSE_WRITE_BACK_DENY_KEYS: Final = frozenset({"id"}) + +_JSON_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) + +# tags metadata rides in as an untyped list; validating through a TypeAdapter (its +# `validate_python` parameter is `Any`, so it accepts an untyped value without complaint) +# resolves it to a concrete element type before use. +_JSON_LIST_ADAPTER: Final = TypeAdapter(list[object]) + +_EMPTY_MAP: Final[Mapping[str, object]] = MappingProxyType({}) + +_EMPTY_STR_MAP: Final[Mapping[str, str]] = MappingProxyType({}) + + +def _dict_of(value: object) -> Mapping[str, object] | None: + return _JSON_DICT_ADAPTER.validate_python(value) if isinstance(value, dict) else None + + +def _custom_metadata(value: object) -> Mapping[str, object] | None: + """A key+team (pre-merged by litellm core), project, or org's custom metadata, minus reserved keys.""" + as_dict: Final = _dict_of(value) + if as_dict is None: + return None + filtered: Final = {k: v for k, v in as_dict.items() if k not in _RESERVED_METADATA_KEYS} + return MappingProxyType(filtered) if filtered else None + + +def _embedded_user_api_key_auth(merged: Mapping[str, object]) -> UserAPIKeyAuth | None: + auth: Final = merged.get("user_api_key_auth") + return auth if isinstance(auth, UserAPIKeyAuth) else None + + +# UserAPIKeyAuth declares these two as a bare `dict | None`, so reading them is itself a +# partially-unknown member access; each accessor isolates that one read behind an explicit +# `object` return type so the rest of this module never touches the untyped field directly. +def _project_metadata_of(auth: UserAPIKeyAuth) -> object: + return auth.project_metadata # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped + + +def _organization_metadata_of(auth: UserAPIKeyAuth) -> object: + return auth.organization_metadata # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped + + +def _jsonable_dict(value: Mapping[str, object]) -> Mapping[str, object]: + """Round-trip through JSON so the payload cannot carry live objects.""" + plain: Final = dict(value) # mutable-ok: json.dumps requires a real dict; consumed immediately + return _JSON_DICT_ADAPTER.validate_python(json.loads(json.dumps(plain, default=str))) + + +def _merged_request_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + { + **(_dict_of(request_data.get("metadata")) or _EMPTY_MAP), + **(_dict_of(request_data.get("litellm_metadata")) or _EMPTY_MAP), + } + ) + + +def _request_metadata(request_data: Mapping[str, object]) -> ThirdlawGuardrailRequestMetadata: + merged: Final = _merged_request_metadata(request_data) + user_fields: Final = MappingProxyType( + {field: value for field in _USER_METADATA_FIELDS if isinstance(value := merged.get(field), str)} + ) + token: Final = merged.get("user_api_key_token") + hash_fallback: Final[Mapping[str, str]] = ( + MappingProxyType({"user_api_key_hash": token}) + if "user_api_key_hash" not in user_fields and isinstance(token, str) + else _EMPTY_STR_MAP + ) + call_id: Final = request_data.get("litellm_call_id") + trace_id: Final = request_data.get("litellm_trace_id") + # litellm_session_id (call-level, header/Anthropic-metadata-derived) and metadata.session_id + # (settable directly by a caller per litellm's own "missing_session_id" documentation) are + # two independent, equally valid sources; litellm core itself treats either as sufficient. + session_id: Final = request_data.get("litellm_session_id") or merged.get("session_id") + model: Final = request_data.get("model") + tags_value: Final = merged.get("tags") + tags_list: Final = _JSON_LIST_ADAPTER.validate_python(tags_value) if isinstance(tags_value, list) else None + tags: Final = tuple(tag for tag in tags_list if isinstance(tag, str)) if tags_list is not None else None + auth_obj: Final = _embedded_user_api_key_auth(merged) + project_metadata: Final = _dict_of(_project_metadata_of(auth_obj)) if auth_obj is not None else None + organization_metadata: Final = _dict_of(_organization_metadata_of(auth_obj)) if auth_obj is not None else None + return ThirdlawGuardrailRequestMetadata( + litellm_version=litellm_version, + litellm_call_id=call_id if isinstance(call_id, str) else None, + litellm_trace_id=trace_id if isinstance(trace_id, str) else None, + litellm_session_id=session_id if isinstance(session_id, str) else None, + model=model if isinstance(model, str) else None, + tags=tags or None, + # Already includes the team's own custom metadata (litellm core layers it on top when + # building this dict), so team_metadata is intentionally not forwarded separately here. + user_api_key_auth_metadata=_custom_metadata(merged.get("user_api_key_auth_metadata")), + project_metadata=_custom_metadata(project_metadata), + organization_metadata=_custom_metadata(organization_metadata), + **user_fields, + **hash_fallback, + ) + + +def _proxy_server_request(request_data: Mapping[str, object]) -> Mapping[str, object]: + return _dict_of(request_data.get("proxy_server_request")) or _EMPTY_MAP + + +def _request_url(request_data: Mapping[str, object]) -> str | None: + url: Final = _proxy_server_request(request_data).get("url") + return url if isinstance(url, str) else None + + +def _redacted_inbound_headers(request_data: Mapping[str, object]) -> Mapping[str, object] | None: + proxy_headers: Final = _dict_of(_proxy_server_request(request_data).get("headers")) + if proxy_headers: + return proxy_headers + return _dict_of(_merged_request_metadata(request_data).get("headers")) or None + + +def _raw_inbound_headers(request_data: Mapping[str, object]) -> Mapping[str, object]: + secret_fields: Final = _dict_of(request_data.get("secret_fields")) or _EMPTY_MAP + return _dict_of(secret_fields.get("raw_headers")) or _EMPTY_MAP + + +def _outbound_request_headers( + request_data: Mapping[str, object], raw_value_header_names: frozenset[str] +) -> Mapping[str, str] | None: + """Every inbound header, using LiteLLM's credential-redacted values; names in + ``raw_value_header_names`` carry the raw value the client sent. + + The proxy strips the header used for LiteLLM auth (e.g. ``authorization``) from its + sanitized copy entirely, so opted-in names are also re-added from the raw headers + rather than only substituted in place. + """ + redacted: Final = _redacted_inbound_headers(request_data) + if redacted is None: + return None + raw_by_lower: Final = MappingProxyType( + {name.lower(): str(value) for name, value in _raw_inbound_headers(request_data).items()} + ) + from_redacted: Final = MappingProxyType( + { + name: ( + raw_by_lower[name.lower()] + if name.lower() in raw_value_header_names and name.lower() in raw_by_lower + else str(value) + ) + for name, value in redacted.items() + } + ) + present_lower: Final = frozenset(name.lower() for name in from_redacted) + reintroduced: Final = MappingProxyType( + { + name: raw_by_lower[name] + for name in raw_value_header_names + if name in raw_by_lower and name not in present_lower + } + ) + return MappingProxyType({**from_redacted, **reintroduced}) + + +def _response_headers(response: object) -> Mapping[str, str] | None: + """The upstream provider's raw HTTP response headers, if ``response`` carries them. + + Every provider transformation stamps these onto ``_hidden_params["additional_headers"]`` + before any post-call hook runs (``process_response_headers`` in core_helpers.py), and the + streaming iterator/wrapper classes (``CustomStreamWrapper``, the Responses and Anthropic + Messages stream wrappers) stamp the same snapshot on themselves at construction, so this + reads identically whether ``response`` is a finished response or the still-open stream. + ``_hidden_params`` is occasionally the ``HiddenParams`` pydantic model rather than a plain + dict, hence the attribute fallback below. + """ + hidden_params: Final[object] = getattr(response, "_hidden_params", None) + headers: Final[object] = ( + hidden_params.get("additional_headers") + if isinstance(hidden_params, Mapping) + else getattr(hidden_params, "additional_headers", None) + ) + if not isinstance(headers, Mapping) or not headers: + return None + return MappingProxyType({str(k): str(v) for k, v in headers.items()}) + + +# Where async_post_call_response_headers_hook stashes response headers for the +# streaming/success hooks to read back. Unprefixed by request_data's other keys +# on purpose -- this is ThirdLaw's own scratch slot, never read or written by core. +_RESPONSE_HEADERS_STASH_KEY: Final = "_thirdlaw_response_headers" + + +def _stashed_response_headers(request_data: Mapping[str, object]) -> Mapping[str, str] | None: + stashed: Final = request_data.get(_RESPONSE_HEADERS_STASH_KEY) + return stashed if isinstance(stashed, Mapping) else None + + +def _request_body(request_data: Mapping[str, object], prefer_snapshot: bool) -> Mapping[str, object] | None: + """Best-effort provider request body; never raises because a guardrail that throws + here would fail live traffic. + + Pre-call must read live ``request_data``: the ``proxy_server_request.body`` snapshot + is taken before pre-call hooks run, so it would resend content an earlier guardrail + (e.g. Presidio masking) already rewrote. + """ + snapshot: Final = _dict_of(_proxy_server_request(request_data).get("body")) if prefer_snapshot else None + source: Final = snapshot if snapshot is not None else request_data + try: + return _jsonable_dict( + MappingProxyType({key: value for key, value in source.items() if key not in _BODY_STRIP_KEYS}) + ) + except Exception: # noqa: BLE001 # best-effort capture; a raise here would fail live traffic + verbose_proxy_logger.warning("ThirdLaw guardrail: could not serialize request body", exc_info=True) + return None + + +def _response_payload(response: object) -> Mapping[str, object] | None: + dump: Final = getattr(response, "model_dump", None) + try: + raw: Final[object] = dump(mode="json") if callable(dump) else response + except Exception: # noqa: BLE001 # best-effort capture; a raise here would fail live traffic + verbose_proxy_logger.warning("ThirdLaw guardrail: could not serialize response body", exc_info=True) + return None + as_dict: Final = _dict_of(raw) + if as_dict is None: + return None + try: + return _jsonable_dict(as_dict) + except Exception: # noqa: BLE001 # best-effort capture; a raise here would fail live traffic + verbose_proxy_logger.warning("ThirdLaw guardrail: could not serialize response body", exc_info=True) + return None + + +def _stream_chunk_payload(item: object) -> Mapping[str, object] | None: + """One buffered stream event as JSON, keeping a sequence number the bridge stamps off-model. + + LiteLLM's chat-to-Responses bridge writes ``sequence_number`` straight onto ``__dict__``, which + ``model_dump`` drops, so it is read back with ``getattr`` and restored on the payload. + """ + dump: Final = getattr(item, "model_dump", None) + if not callable(dump): + as_dict: Final = _dict_of(item) + return _jsonable_dict(as_dict) if as_dict is not None else None + try: + dumped: Final[object] = dump(mode="json") + except Exception: # noqa: BLE001 # one unserializable event must not drop the whole stream from the payload + return None + dumped_dict: Final = _dict_of(dumped) + if dumped_dict is None: + return None + sequence: Final = getattr(item, "sequence_number", None) + if isinstance(sequence, int) and "sequence_number" not in dumped_dict: + return _jsonable_dict(MappingProxyType({**dumped_dict, "sequence_number": sequence})) + return _jsonable_dict(dumped_dict) + + +def _is_unreachable_error(error: Exception) -> bool: + if isinstance(error, httpx.HTTPStatusError): + return error.response.status_code in _UNREACHABLE_STATUS_CODES + return isinstance(error, (httpx.RequestError, LitellmTimeout)) + + +def _service_error_message(error: Exception) -> str: + """Prefer ThirdLaw's own explanation over httpx's generic status text. + + ThirdLaw returns ``{"detail": {"error", "reason", "message"}}`` on a fail-closed + response. Anything else (an ingress 502, a non-JSON body, an older service version) + falls back to the status line. + """ + if not isinstance(error, httpx.HTTPStatusError): + return str(error) + try: + detail: Final[object] = error.response.json().get("detail") + except Exception: # noqa: BLE001 # non-JSON or non-object error body: keep the status line + return str(error) + if isinstance(detail, str) and detail: + return detail + detail_fields: Final = _dict_of(detail) + if detail_fields is None: + return str(error) + message: Final = detail_fields.get("message") + if not isinstance(message, str) or not message: + return str(error) + reason: Final = detail_fields.get("reason") + return f"{message} (reason={reason})" if isinstance(reason, str) and reason else message + + +def _decision_trace(decision: ThirdlawGuardrailResponse) -> Mapping[str, object]: + return MappingProxyType( + { + "action": decision.action, + "message": decision.message, + "response_status": decision.response_status, + "modified_request": decision.request_body is not None, + "modified_response": decision.response_body is not None, + } + ) + + +class ThirdlawGuardrailMissingConfig(ValueError): + pass + + +class ThirdlawGuardrail(CustomGuardrail): + def __init__( + self, + api_base: str | None = None, + api_key: str | None = None, + additional_headers: str | None = None, + guardrail_timeout: int | None = 60, + streaming_buffer_until_moderated: bool = True, + streaming_end_of_stream_only: bool = True, + streaming_sampling_rate: int = 5, + unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", + unscannable_stream_fallback: Literal["fail_closed", "fail_open"] = "fail_closed", + send_stream_chunks: bool = False, + additional_provider_specific_params: Mapping[str, object] | None = None, + headers: Mapping[str, str] | None = None, + extra_headers: Sequence[str] | None = None, + async_handler: AsyncHTTPHandler | None = None, + **kwargs, # noqa: ANN003 # kwargs-ok: forwarded verbatim to CustomGuardrail, which owns their types + ) -> None: + resolved_base: Final = api_base or get_secret_str("THIRDLAW_API_BASE") + if not resolved_base: + raise ThirdlawGuardrailMissingConfig( + "ThirdLaw api_base is required. Set api_base in the guardrail " + "config or the THIRDLAW_API_BASE environment variable." + ) + if streaming_sampling_rate < 1: + raise ValueError(f"streaming_sampling_rate must be >= 1 (got {streaming_sampling_rate})") + + trimmed_base: Final = resolved_base.rstrip("/") + self.api_base = trimmed_base if trimmed_base.endswith(_ENDPOINT_PATH) else f"{trimmed_base}{_ENDPOINT_PATH}" + + resolved_key: Final = api_key or get_secret_str("THIRDLAW_API_KEY") + auth_header: Final[Mapping[str, str]] = ( + MappingProxyType({"Authorization": f"Bearer {resolved_key}"}) if resolved_key else _EMPTY_STR_MAP + ) + self.http_headers: Mapping[str, str] = MappingProxyType( + {"Content-Type": "application/json", **(headers or _EMPTY_STR_MAP), **auth_header} + ) + + configured_names: Final = tuple(additional_headers.split(",")) if additional_headers else () + self.raw_value_header_names: frozenset[str] = frozenset( + stripped.lower() for name in (*configured_names, *(extra_headers or ())) if (stripped := name.strip()) + ) + + self.guardrail_timeout = httpx.Timeout(timeout=guardrail_timeout or 60, connect=5.0) + self.streaming_buffer_until_moderated = streaming_buffer_until_moderated + self.streaming_end_of_stream_only = streaming_end_of_stream_only + self.streaming_sampling_rate = streaming_sampling_rate + self.unreachable_fallback: Literal["fail_closed", "fail_open"] = unreachable_fallback + self.unscannable_stream_fallback: Literal["fail_closed", "fail_open"] = unscannable_stream_fallback + self.send_stream_chunks: bool = send_stream_chunks + self.additional_provider_specific_params: Mapping[str, object] = ( + additional_provider_specific_params or _EMPTY_MAP + ) + + kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) # mutable-ok: base API + super().__init__(**kwargs) + + self.async_handler = async_handler or get_async_httpx_client( + llm_provider=httpxSpecialProvider.GuardrailCallback, + params={"timeout": self.guardrail_timeout}, # mutable-ok: one-shot client-factory argument + ) + + @classmethod + def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: # mutable-ok: CustomGuardrail base-class contract + return [ # mutable-ok: CustomGuardrail base-class contract returns a list + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + GuardrailEventHooks.during_call, + ] + + @staticmethod + def get_config_model() -> type["GuardrailConfigModel"] | None: + from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrailConfigModel, + ) + + return ThirdlawGuardrailConfigModel + + def _build_wire_request( + self, + *, + wire_event: _WireEvent, + request_data: dict[str, object], # mutable-ok: proxy-shared request dict, forwarded to base-class helpers + response_body: Mapping[str, object] | None, + response_headers: Mapping[str, str] | None = None, + streamed_deltas: Sequence[str] | None = None, + response_chunks: Sequence[Mapping[str, object]] | None = None, + response_sse: str | None = None, + ) -> ThirdlawGuardrailRequest: + dynamic_params: Final = _JSON_DICT_ADAPTER.validate_python( + self.get_guardrail_dynamic_request_body_params(request_data) + ) + combined_params: Final = MappingProxyType({**self.additional_provider_specific_params, **dynamic_params}) + return ThirdlawGuardrailRequest( + event_type=wire_event, + metadata=_request_metadata(request_data), + request_url=_request_url(request_data), + request_headers=_outbound_request_headers(request_data, self.raw_value_header_names), + request_body=_request_body(request_data, prefer_snapshot=wire_event != "pre_call"), + response_body=response_body, + response_headers=response_headers, + response_chunks=tuple(response_chunks) if response_chunks else None, + response_sse=response_sse or None, + streamed_deltas_not_in_body=tuple(streamed_deltas) if streamed_deltas else None, + additional_provider_specific_params=combined_params or None, + ) + + def _record_trace( + self, + *, + request_data: dict[str, object], # mutable-ok: the trace helper appends to this dict's metadata bucket + event_type: GuardrailEventHooks, + status: GuardrailStatus, + started_at: datetime, + trace: Mapping[str, object], + ) -> None: + now: Final = datetime.now(timezone.utc) + self.add_standard_logging_guardrail_information_to_request_data( + guardrail_json_response=dict(trace), # mutable-ok: the logging helper requires a plain dict + request_data=request_data, + guardrail_status=status, + start_time=started_at.timestamp(), + end_time=now.timestamp(), + duration=(now - started_at).total_seconds(), + event_type=event_type, + ) + + def _handle_call_failure(self, *, error: Exception, wire_event: _WireEvent) -> None: + """Fail-open logs and returns; everything else raises.""" + if _is_unreachable_error(error) and self.unreachable_fallback == "fail_open": + verbose_proxy_logger.critical( + "ThirdLaw guardrail unreachable (fail-open). Proceeding without guardrail. " + "guardrail_name=%s api_base=%s event=%s", + self.guardrail_name, + self.api_base, + wire_event, + exc_info=error, + ) + return + raise GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message=f"ThirdLaw guardrail request failed: {_service_error_message(error)}", + ) from error + + async def _run_thirdlaw( + self, + *, + event_type: GuardrailEventHooks, + wire_event: _WireEvent, + request_data: dict[str, object], # mutable-ok: proxy-shared request dict; trace records land in it + response_body: Mapping[str, object] | None = None, + response_headers: Mapping[str, str] | None = None, + streamed_deltas: Sequence[str] | None = None, + response_chunks: Sequence[Mapping[str, object]] | None = None, + response_sse: str | None = None, + ) -> ThirdlawGuardrailResponse | None: + """POST the full payload to ThirdLaw and return its decision. + + Returns None when the service is unreachable and the guardrail is configured + fail-open; raises for every other failure and records the guardrail trace on + all paths (the decorator on the lifecycle hooks skips its auto-record when an + entry was already written here). + """ + started_at: Final = datetime.now(timezone.utc) + payload: Final = self._build_wire_request( + wire_event=wire_event, + request_data=request_data, + response_body=response_body, + response_headers=response_headers, + streamed_deltas=streamed_deltas, + response_chunks=response_chunks, + response_sse=response_sse, + ) + try: + http_response: Final = await self.async_handler.post( + url=self.api_base, + headers=dict(self.http_headers), # mutable-ok: the HTTP client requires a plain dict + json=payload.model_dump(mode="json", exclude_none=True), + ) + http_response.raise_for_status() + decision: Final = ThirdlawGuardrailResponse.model_validate(http_response.json()) + except Exception as error: # noqa: BLE001 # every transport/parse failure funnels into the fallback policy + self._record_trace( + request_data=request_data, + event_type=event_type, + status="guardrail_failed_to_respond", + started_at=started_at, + trace={"error": _service_error_message(error)}, # mutable-ok: one-shot trace payload + ) + self._handle_call_failure(error=error, wire_event=wire_event) + return None + status: Final[GuardrailStatus] = "guardrail_intervened" if decision.action == "block" else "success" + self._record_trace( + request_data=request_data, + event_type=event_type, + status=status, + started_at=started_at, + trace=_decision_trace(decision), + ) + return decision + + def _block_status(self, response_status: int | None) -> int: + """A block is a refusal, so it can only travel as a 4xx/5xx. + + Honoring a success status the service sent would hand the caller a 200 that + reads as an allow. + """ + if response_status is None: + return 400 + if 400 <= response_status <= 599: + return response_status + verbose_proxy_logger.warning( + "ThirdLaw guardrail: ignoring non-error response_status %s on a block decision; using 400", + response_status, + ) + return 400 + + def _block_exception(self, decision: ThirdlawGuardrailResponse) -> GuardrailRaisedException: + return GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message=decision.message or "Content violates ThirdLaw policy", + should_wrap_with_default_message=False, + status_code=self._block_status(decision.response_status), + blocked_content=True, + ) + + def _applied_request_modifications( + self, *, data: Mapping[str, object], decision: ThirdlawGuardrailResponse + ) -> dict[str, object]: # mutable-ok: the pre-call hook contract returns the (new) request dict + replacement: Final = decision.request_body + if not replacement: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: modify_request decision carried no request_body; request unchanged" + ) + return dict(data) # mutable-ok: the pre-call hook contract returns a plain request dict + accepted: Final = { # mutable-ok: one-shot filter merged into the returned request dict + key: value for key, value in replacement.items() if key not in _WRITE_BACK_DENY_KEYS + } + denied: Final = replacement.keys() - accepted.keys() + if denied: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: dropping protected keys from modify_request write-back: %s", + sorted(denied), + ) + return {**data, **accepted} # mutable-ok: the proxy owns the replaced request dict + + @staticmethod + def _carry_hidden_params(*, source: object, target: object) -> None: + """model_validate() builds a fresh instance with no memory of the source's private + attributes -- ``_hidden_params`` (which carries ``additional_headers`` for response- + header forwarding) is one of those, so copy it across explicitly. + """ + hidden_params = getattr(source, "_hidden_params", None) + if hidden_params is not None: + setattr(target, "_hidden_params", hidden_params) # target's concrete type varies by call site + + def _without_response_id(self, replacement: Mapping[str, object]) -> Mapping[str, object]: + """Drop ``id`` from a modify_response write-back. + + litellm hands out an encrypted response id that the client sends back as + previous_response_id and that carries the deployment the turn was routed to, so a + rewrite that renames it breaks both continuation and routing. + """ + if not _RESPONSE_WRITE_BACK_DENY_KEYS & replacement.keys(): + return replacement + verbose_proxy_logger.warning( + "ThirdLaw guardrail: ignoring %s in a modify_response; the response id is encrypted " + "and the client chains the next turn off it", + sorted(_RESPONSE_WRITE_BACK_DENY_KEYS & replacement.keys()), + ) + return MappingProxyType( + {key: value for key, value in replacement.items() if key not in _RESPONSE_WRITE_BACK_DENY_KEYS} + ) + + def _modified_response(self, *, response: object, replacement: Mapping[str, object]) -> object: + replacement = self._without_response_id(replacement) # rebind-ok: one guard for every shape below + if isinstance(response, ModelResponse): + # ModelResponse validation coerces a non-list ``choices`` into a single + # empty choice instead of raising, which would silently blank the response. + if "choices" in replacement and not isinstance(replacement.get("choices"), list): + raise GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message="ThirdLaw guardrail returned a malformed modified response: choices must be a list", + ) + merged: Final = { # mutable-ok: one-shot overlay consumed immediately by model_validate + **_JSON_DICT_ADAPTER.validate_python(response.model_dump()), + **replacement, + } + try: + validated: Final = ModelResponse.model_validate(merged) + except ValidationError as error: + raise GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message=f"ThirdLaw guardrail returned a malformed modified response: {error}", + ) from error + self._carry_hidden_params(source=response, target=validated) + return validated + response_dict: Final = _dict_of(response) + if response_dict is not None: + return {**response_dict, **replacement} # mutable-ok: the proxy owns the replaced response dict + if isinstance(response, BaseModel): + merged_body: Final = { # mutable-ok: one-shot overlay consumed immediately by model_validate + **_JSON_DICT_ADAPTER.validate_python(response.model_dump(mode="json")), + **replacement, + } + try: + revalidated: Final = type(response).model_validate(merged_body) + except ValidationError as error: + raise GuardrailRaisedException( + guardrail_name=self.guardrail_name, + message=f"ThirdLaw guardrail returned a malformed modified response: {error}", + ) from error + self._carry_hidden_params(source=response, target=revalidated) + return revalidated + verbose_proxy_logger.warning( + "ThirdLaw guardrail: modify_response is not supported for %s responses; returning original", + type(response).__name__, + ) + return response + + def _mark_applied(self, request_data: dict[str, object]) -> None: # mutable-ok: header helper appends to metadata + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) + + add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) + + async def async_post_call_response_headers_hook( + self, + data: dict[str, object], # mutable-ok: proxy-shared request dict, per the CustomLogger hook contract + user_api_key_dict: UserAPIKeyAuth, + response: object, + request_headers: Mapping[str, str] | None = None, + litellm_call_info: Mapping[str, object] | None = None, + ) -> Mapping[str, str] | None: + """Capture the provider's response headers off the true, unwrapped response. + + A guardrail chained ahead of ThirdLaw can obscure ``_hidden_params`` on the + ``response`` the streaming hook later receives (``ResponsesIDSecurity``, e.g., + hands back a bare generator with none); this hook always gets the real thing. + Always returns None -- it only stashes for ThirdLaw's own later use. + """ + headers: Final = _response_headers(response) + if headers is not None: + data[_RESPONSE_HEADERS_STASH_KEY] = headers + return None + + @log_guardrail_information + async def async_pre_call_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + cache: DualCache, + data: dict[str, object], # mutable-ok: proxy-shared request dict, per the CustomLogger hook contract + call_type: CallTypesLiteral, + ) -> dict[str, object]: # mutable-ok: the proxy replaces its request dict with this return value + data.pop(_RESPONSE_HEADERS_STASH_KEY, None) # strip any caller-forged value; always runs first + decision: Final = await self._run_thirdlaw( + event_type=GuardrailEventHooks.pre_call, wire_event="pre_call", request_data=data + ) + if decision is None: + return data + if decision.action == "block": + raise self._block_exception(decision) + self._mark_applied(data) + if decision.action == "modify_request": + return self._applied_request_modifications(data=data, decision=decision) + if decision.action == "modify_response": + verbose_proxy_logger.warning("ThirdLaw guardrail: ignoring modify_response decision on pre_call") + return data + + @log_guardrail_information + async def async_moderation_hook( + self, + data: dict[str, object], # mutable-ok: proxy-shared request dict, per the CustomLogger hook contract + user_api_key_dict: UserAPIKeyAuth, + call_type: CallTypesLiteral, + ) -> dict[str, object]: # mutable-ok: the hook contract returns the request dict + decision: Final = await self._run_thirdlaw( + event_type=GuardrailEventHooks.during_call, wire_event="during_call", request_data=data + ) + if decision is None: + return data + if decision.action == "block": + raise self._block_exception(decision) + if decision.action != "allow": + verbose_proxy_logger.warning( + "ThirdLaw guardrail: %s decision cannot be applied on during_call (runs parallel to the LLM call)", + decision.action, + ) + self._mark_applied(data) + return data + + @log_guardrail_information + async def async_post_call_success_hook( + self, + data: dict[str, object], # mutable-ok: proxy-shared request dict, per the CustomLogger hook contract + user_api_key_dict: UserAPIKeyAuth, + response: LLMResponseTypes, + ) -> LLMResponseTypes: + response_body: Final = _response_payload(response) + if response_body is None: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: skipping post_call scan for unsupported response type %s", + str(type(response).__name__), + ) + return response + decision: Final = await self._run_thirdlaw( + event_type=GuardrailEventHooks.post_call, + wire_event="post_call", + request_data=data, + response_body=response_body, + response_headers=_stashed_response_headers(data) or _response_headers(response), + ) + if decision is None: + return response + if decision.action == "block": + raise self._block_exception(decision) + self._mark_applied(data) + if decision.action != "modify_response" or not decision.response_body: + return response + if self.run_in_parallel: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: modify_response is discarded when run_in_parallel=True (block-only mode)" + ) + modified: Final = self._modified_response(response=response, replacement=decision.response_body) + return cast(LLMResponseTypes, modified) # cast-ok: dict passthrough mirrors the proxy /v1/messages contract + + async def async_post_call_streaming_iterator_hook( + self, + user_api_key_dict: UserAPIKeyAuth, + response: AsyncIterator[object], + request_data: dict[str, object], # mutable-ok: proxy-shared request dict; trace records land in it + ) -> AsyncGenerator[ModelResponseStream, None]: + buffer: Final = self._buffer_until_moderated() + end_of_stream_only: Final = buffer or self.streaming_end_of_stream_only + # Prefer the stash (see async_post_call_response_headers_hook); `response` here + # can be a bare generator with no _hidden_params once another callback is + # chained ahead of ThirdLaw. Falls back to `response` when ThirdLaw runs alone. + response_headers: Final = _stashed_response_headers(request_data) or _response_headers(response) + iterator: Final = ( + self._end_of_stream_moderated_stream( + response=response, request_data=request_data, buffer=buffer, response_headers=response_headers + ) + if end_of_stream_only + else self._sampled_stream(response=response, request_data=request_data, response_headers=response_headers) + ) + async for item in iterator: + # Raw-SSE frames (bytes) ride through the same pipe; the proxy's data + # generator forwards str/bytes chunks verbatim, matching bedrock's hook. + yield cast(ModelResponseStream, item) # cast-ok: raw-SSE byte frames share the typed stream (bedrock) + + @staticmethod + def _assembled_stream_response(collected: Sequence[object], surface: StreamSurface) -> _AssembledStream | None: + """Assemble the buffered stream into the scannable body its surface produces.""" + match surface: + case StreamSurface.ANTHROPIC_MESSAGES: + return assemble_anthropic_sse_body(collected) + case StreamSurface.RESPONSES: + return ThirdlawGuardrail._assembled_responses_stream_response(collected) + case StreamSurface.OPAQUE_SSE: + return None + case StreamSurface.CHAT_COMPLETIONS: + return ThirdlawGuardrail._assembled_chat_stream_response(collected) + + @staticmethod + def _assembled_responses_stream_response(collected: Sequence[object]) -> ResponsesAPIResponse | None: + """The finished Responses body, which is the shape the non-streaming route already posts.""" + return final_responses_api_response(collected) + + @staticmethod + def _streamed_deltas_not_in_body( + collected: Sequence[object], assembled: _AssembledStream, surface: StreamSurface + ) -> tuple[str, ...]: + """Delta text the client already received that the scanned body does not carry. + + Reasoning summaries and tool-call arguments reach a /v1/responses client through delta + events some providers never repeat in the terminal body. Posting them beside the body + lets the scan cover the whole turn without widening the body contract. + """ + if surface is not StreamSurface.RESPONSES or not isinstance(assembled, ResponsesAPIResponse): + return () + return responses_deltas_absent_from_body(collected, assembled) + + def _stream_payload( + self, collected: Sequence[object], surface: StreamSurface + ) -> tuple[tuple[Mapping[str, object], ...] | None, str | None]: + """The buffered stream as the service receives it beside the assembled body. + + The body is what LiteLLM folded; the stream is what the client received. Posting both lets + the service fold on its own side and fall back to whichever side has no gap, with no + LiteLLM release in between. Returns ``(chunks, sse_text)``; at most one is set. + """ + if not self.send_stream_chunks: + return None, None + match surface: + case StreamSurface.ANTHROPIC_MESSAGES: + return None, sse_stream_text(collected) + case StreamSurface.RESPONSES | StreamSurface.CHAT_COMPLETIONS: + chunks: Final = tuple( + payload for item in collected if (payload := _stream_chunk_payload(item)) is not None + ) + return (chunks or None), None + case StreamSurface.OPAQUE_SSE: + return None, None + + @staticmethod + def _assembled_chat_stream_response(collected: Sequence[object]) -> ModelResponse | None: + from litellm.main import stream_chunk_builder + + try: + assembled: Final = stream_chunk_builder(chunks=list(collected)) # mutable-ok: builder requires a list + except Exception: # noqa: BLE001 # unassembleable streams route to the explicit fail-closed/pass-through path + verbose_proxy_logger.warning("ThirdLaw guardrail: could not assemble streamed response", exc_info=True) + return None + return assembled if isinstance(assembled, ModelResponse) else None + + def _buffer_until_moderated(self) -> bool: + """Whether to withhold the stream until the scan decides. + + Buffering replays the original chunks on release, so a guardrail asked to mask content would + hand back the very text it was told to redact. Masking therefore wins over buffering. + """ + requested: Final = self.streaming_buffer_until_moderated + if requested and self.mask_response_content: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: streaming_buffer_until_moderated is disabled for %s because " + "mask_response_content=True -- buffered replay would release unredacted original chunks", + self.guardrail_name, + ) + return False + return requested + + def _streaming_block_error(self, message: str) -> Exception: + from litellm.proxy.proxy_server import StreamingCallbackError + + return StreamingCallbackError(message) + + @staticmethod + def _stream_error_items( + message: str, surface: StreamSurface, collected: Sequence[object] + ) -> Sequence[object] | None: + """Terminal stream items framing a refusal in this surface's wire format. + + ``None`` means the surface cannot frame its own error, so the caller raises instead and + lets the proxy's data generator serialize it. + """ + match surface: + case StreamSurface.ANTHROPIC_MESSAGES: + return anthropic_sse_error_frames(message) + case StreamSurface.RESPONSES: + return ThirdlawGuardrail._responses_error_events(message, collected) + case StreamSurface.CHAT_COMPLETIONS | StreamSurface.OPAQUE_SSE: + return None + + @staticmethod + def _responses_error_events(message: str, collected: Sequence[object]) -> Sequence[object]: + """A ``/v1/responses`` refusal as an in-stream error event. + + A raised exception reaches the client as the proxy's ``{"error": ...}`` blob, which carries + no top-level ``type`` and so is not a Responses event at all. ``collected`` is forwarded so + the error event continues the stream's sequence numbering rather than restarting at zero. + """ + from fastapi import HTTPException + + from litellm.llms.openai.responses.guardrail_translation.handler import ( + OpenAIResponsesHandler, + ) + + return ( + OpenAIResponsesHandler().build_stream_error_items( + HTTPException(status_code=400, detail=message), + responses_so_far=collected, + ) + or () + ) + + async def _end_of_stream_moderated_stream( + self, + *, + response: AsyncIterator[object], + request_data: dict[str, object], # mutable-ok: proxy-shared request dict; trace records land in it + buffer: bool, + response_headers: Mapping[str, str] | None, + ) -> AsyncGenerator[object, None]: + started: Final = time.monotonic() + collected: Final[list[object]] = [] # mutable-ok: streaming chunk buffer + async for item in response: + collected.append(item) + if not buffer: + yield item + + surface: Final = classify_stream(collected) + assembled: Final = self._assembled_stream_response(collected, surface) + if assembled is None: + async for item in self._handle_unassembleable(collected=collected, surface=surface, buffer=buffer): + yield item + return + + stream_chunks, stream_sse = self._stream_payload(collected, surface) + try: + decision: Final = await self._run_thirdlaw( + event_type=GuardrailEventHooks.post_call, + wire_event="post_call", + request_data=request_data, + response_body=_response_payload(assembled), + response_headers=response_headers, + streamed_deltas=self._streamed_deltas_not_in_body(collected, assembled, surface), + response_chunks=stream_chunks, + response_sse=stream_sse, + ) + except Exception as error: # noqa: BLE001 # after keepalive flush a raise cannot reach the client; send a frame + flushed_frames: Final = ( + self._stream_error_items( + f"ThirdLaw guardrail request failed: {_service_error_message(error)}", surface, collected + ) + if self._sse_headers_flushed(started) + else None + ) + if flushed_frames is None: + raise + for frame in flushed_frames: + yield frame + return + async for item in self._emit_end_of_stream_outcome( + decision=decision, assembled=assembled, collected=collected, surface=surface, buffer=buffer + ): + yield item + + async def _emit_end_of_stream_outcome( + self, + *, + decision: ThirdlawGuardrailResponse | None, + assembled: _AssembledStream, + collected: Sequence[object], + surface: StreamSurface, + buffer: bool, + ) -> AsyncGenerator[object, None]: + if decision is None or decision.action in ("allow", "modify_request"): + if decision is not None and decision.action == "modify_request": + verbose_proxy_logger.warning("ThirdLaw guardrail: ignoring modify_request decision on a stream") + if buffer: + for item in collected: + yield item + return + + if decision.action == "block": + message: Final = decision.message or "Content violates ThirdLaw policy" + block_items: Final = self._stream_error_items(message, surface, collected) + if block_items is None: + raise self._streaming_block_error(message) + for item in block_items: + yield item + return + + if not buffer: + verbose_proxy_logger.warning( + "ThirdLaw guardrail: modify_response arrived after chunks were already delivered " + "(streaming_buffer_until_moderated=False); response not modified" + ) + return + if not decision.response_body: + for item in collected: + yield item + return + async for item in self._emit_modified_stream( + assembled=assembled, replacement=decision.response_body, surface=surface + ): + yield item + + async def _handle_unassembleable( + self, + *, + collected: Sequence[object], + surface: StreamSurface, + buffer: bool, + ) -> AsyncGenerator[object, None]: + """Refuse a stream that could not be assembled for scanning. + + A stream whose surface has no assembler, or that died before carrying a body, is refused + unless ``unscannable_stream_fallback`` is ``fail_open``, because forwarding it unscanned + lets a caller pick an endpoint to dodge the guardrail. + """ + if is_terminal_error_stream(collected): + # Only the refusal an earlier guardrail in the chain already emitted; replacing it + # would hide the rejection the client is owed. + for item in collected: + yield item + return + refusal: Final = f"{self.guardrail_name}: streamed response could not be assembled for scanning, blocking it" + if any(isinstance(item, ModelResponseStream) for item in collected): + # Real chat-completions chunks that will not assemble are a failure to scan a supported + # shape, not an unsupported one, so the fail_open opt-in does not cover them. + raise self._streaming_block_error(refusal) + # Only the explicit opt-in opens: Literal is not enforced at runtime, so a typo + # in the config must fail closed rather than silently disable the scan. + if self.unscannable_stream_fallback == "fail_open": + verbose_proxy_logger.warning( + "ThirdLaw guardrail: unsupported stream shape passed through unscanned " + "(unscannable_stream_fallback=fail_open)" + ) + if buffer: + for item in collected: + yield item + return + refusal_items: Final = self._stream_error_items(refusal, surface, collected) + if refusal_items is None: + raise self._streaming_block_error(refusal) + for item in refusal_items: + yield item + + async def _emit_modified_stream( + self, + *, + assembled: _AssembledStream, + replacement: Mapping[str, object], + surface: StreamSurface, + ) -> AsyncGenerator[object, None]: + if surface is StreamSurface.RESPONSES: + async for event in self._emit_modified_responses_stream(assembled=assembled, replacement=replacement): + yield event + return + if surface is StreamSurface.ANTHROPIC_MESSAGES: + for frame in self._modified_anthropic_frames(assembled=assembled, replacement=replacement): + yield frame + return + modified: Final = self._modified_response(response=assembled, replacement=replacement) + if not isinstance(modified, ModelResponse): + raise self._streaming_block_error(f"{self.guardrail_name}: modified streamed response failed validation") + from litellm.llms.base_llm.base_model_iterator import MockResponseIterator + + async for chunk in MockResponseIterator(model_response=modified): + yield chunk + + def _modified_anthropic_frames( + self, *, assembled: _AssembledStream, replacement: Mapping[str, object] + ) -> Sequence[bytes]: + """Re-emit a rewritten Messages body as the SSE frames its client expects. + + A chat-shaped rewrite fails closed rather than being merged: the overlay is a shallow + merge, so ``choices`` would land beside the untouched ``content`` and the stream would + re-emit the very text the rewrite asked to redact. + """ + if "choices" in replacement: + raise self._streaming_block_error( + f"{self.guardrail_name}: modify_response answered a /v1/messages stream with " + '"choices"; the Messages body carries its text in "content"' + ) + modified: Final = self._modified_response(response=assembled, replacement=replacement) + body: Final = _dict_of(modified) + if body is None: + raise self._streaming_block_error(f"{self.guardrail_name}: modified streamed response failed validation") + return anthropic_sse_chunks_from_body(body) + + async def _emit_modified_responses_stream( + self, + *, + assembled: _AssembledStream, + replacement: Mapping[str, object], + ) -> AsyncGenerator[object, None]: + """Re-emit a rewritten Responses body as the events its client expects.""" + from litellm.responses.streaming_iterator import ( + MockResponsesAPIStreamingIterator, + build_synthetic_response_events, + ) + + modified: Final = self._modified_response(response=assembled, replacement=replacement) + if not isinstance(modified, ResponsesAPIResponse): + raise self._streaming_block_error(f"{self.guardrail_name}: modified streamed response failed validation") + for event in build_synthetic_response_events( + transformed=modified, + logging_obj=None, + chunk_size=MockResponsesAPIStreamingIterator.CHUNK_SIZE, + ): + yield event + + async def _sampled_stream( + self, + *, + response: AsyncIterator[object], + request_data: dict[str, object], # mutable-ok: proxy-shared request dict; trace records land in it + response_headers: Mapping[str, str] | None, + ) -> AsyncGenerator[object, None]: + sampling_rate: Final = self.streaming_sampling_rate + collected: Final[list[object]] = [] # mutable-ok: streaming chunk buffer + async for item in response: + collected.append(item) + yield item + if len(collected) % sampling_rate != 0 or classify_stream(collected) is not StreamSurface.CHAT_COMPLETIONS: + continue + interim = self._assembled_stream_response(collected, StreamSurface.CHAT_COMPLETIONS) + if interim is None: + continue + interim_decision = await self._run_thirdlaw( + event_type=GuardrailEventHooks.post_call, + wire_event="post_call", + request_data=request_data, + response_body=_response_payload(interim), + response_headers=response_headers, + ) + if interim_decision is not None and interim_decision.action == "block": + raise self._streaming_block_error(interim_decision.message or "Content violates ThirdLaw policy") + + surface: Final = classify_stream(collected) + if surface is StreamSurface.RESPONSES: + # Interim scans need an assembled body and a Responses stream carries none until its + # terminal event, so nothing was scanned until every chunk had already been delivered. + verbose_proxy_logger.warning( + "ThirdLaw guardrail: a /v1/responses stream was scanned only once its turn " + "finished, because interim scans have no body to read; set " + "streaming_buffer_until_moderated=True to withhold it until the scan decides" + ) + assembled: Final = self._assembled_stream_response(collected, surface) + if assembled is None: + async for item in self._handle_unassembleable(collected=collected, surface=surface, buffer=False): + yield item + return + stream_chunks, stream_sse = self._stream_payload(collected, surface) + final_decision: Final = await self._run_thirdlaw( + event_type=GuardrailEventHooks.post_call, + wire_event="post_call", + request_data=request_data, + response_body=_response_payload(assembled), + response_headers=response_headers, + streamed_deltas=self._streamed_deltas_not_in_body(collected, assembled, surface), + response_chunks=stream_chunks, + response_sse=stream_sse, + ) + if final_decision is None: + return + if final_decision.action == "block": + message: Final = final_decision.message or "Content violates ThirdLaw policy" + block_items: Final = self._stream_error_items(message, surface, collected) + if block_items is None: + raise self._streaming_block_error(message) + for item in block_items: + yield item + return + if final_decision.action == "modify_response": + verbose_proxy_logger.warning( + "ThirdLaw guardrail: modify_response arrived after chunks were already delivered " + "(streaming_end_of_stream_only=False); response not modified" + ) + + @staticmethod + def _sse_headers_flushed(started_monotonic: float) -> bool: + from litellm.proxy.common_utils.sse_keepalive import keepalive_ping_has_fired + + return keepalive_ping_has_fired( + time.monotonic() - started_monotonic, litellm.anthropic_sse_ping_interval_seconds + ) diff --git a/litellm/proxy/guardrails/stream_surface.py b/litellm/proxy/guardrails/stream_surface.py new file mode 100644 index 00000000000..3167940bda1 --- /dev/null +++ b/litellm/proxy/guardrails/stream_surface.py @@ -0,0 +1,149 @@ +"""Wire format of a buffered streaming response, for guardrails that scan whole streams. + +A `post_call` guardrail receives a stream as whatever its route produces: chat-completions chunk +objects, raw Anthropic SSE bytes, Responses API event models, or opaque SSE from a route that +frames its own. Each has to be read, refused and rewritten in its own shape, so a guardrail that +branches on a single "is this raw SSE" flag silently treats three of those four as the fourth. + +Classification lives here rather than in each guardrail so the surfaces stay one list. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from enum import Enum, auto +from typing import Final + +from litellm.proxy.guardrails.anthropic_sse import ( + is_anthropic_sse_stream, + is_raw_sse_stream, + is_sse_error_stream, +) +from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamEvents + +_RESPONSES_EVENT_TYPE_PREFIX: Final = "response." + +# Only these carry the finished output; response.created also carries a body, but an empty one +_RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset( + { + ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value, + ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value, + ResponsesAPIStreamEvents.RESPONSE_FAILED.value, + } +) + +# Every event whose ``delta`` is model output already on its way to the client. Read off the event +# enum rather than listed, so an event added there cannot quietly fall out of the comparison +_RESPONSES_DELTA_EVENT_TYPES: Final = frozenset( + event.value for event in ResponsesAPIStreamEvents if event.value.endswith(".delta") +) + +# What makes two delta events part of the same field of the turn, rather than two fields that +# merely streamed next to each other +_RESPONSES_DELTA_FIELD_ATTRS: Final = ("type", "item_id", "output_index", "content_index", "summary_index") + + +class StreamSurface(Enum): + """Which endpoint shape a buffered stream belongs to.""" + + CHAT_COMPLETIONS = auto() + ANTHROPIC_MESSAGES = auto() + RESPONSES = auto() + OPAQUE_SSE = auto() + + +def _stream_item_field(item: object, field: str) -> object | None: + return item.get(field) if isinstance(item, dict) else getattr(item, field, None) + + +def _stream_item_type(item: object) -> str | None: + event_type: Final = _stream_item_field(item, "type") + return event_type if isinstance(event_type, str) else None + + +def classify_stream(all_chunks: Sequence[object]) -> StreamSurface: + """Wire format the buffered chunks belong to. + + Raw bytes are not enough to name the surface: ``/v1/messages`` frames its own SSE, but so does + the Google ``:streamGenerateContent`` route, and reading the latter as Anthropic would refuse it + in a format its client cannot parse. + """ + if is_raw_sse_stream(all_chunks): + return StreamSurface.ANTHROPIC_MESSAGES if is_anthropic_sse_stream(all_chunks) else StreamSurface.OPAQUE_SSE + if any( + (event_type := _stream_item_type(chunk)) is not None and event_type.startswith(_RESPONSES_EVENT_TYPE_PREFIX) + for chunk in all_chunks + ): + return StreamSurface.RESPONSES + return StreamSurface.CHAT_COMPLETIONS + + +def final_responses_api_response(all_chunks: Sequence[object]) -> ResponsesAPIResponse | None: + """The finished body a terminal ``/v1/responses`` event carries. + + Read with ``getattr`` rather than ``model_dump``: litellm's chat-to-Responses bridge stamps + ``sequence_number`` straight onto ``__dict__``, which pydantic's dump path drops. + + A stream cut short before it completes has to read as unassembled rather than as a clean empty + response, since ``response.created`` carries a body too and scanning that would release every + buffered delta unscanned. + """ + return next( + ( + body + for chunk in reversed(all_chunks) + if _stream_item_field(chunk, "type") in _RESPONSES_TERMINAL_EVENT_TYPES + and isinstance(body := _stream_item_field(chunk, "response"), ResponsesAPIResponse) + ), + None, + ) + + +def _responses_delta_field(chunk: object) -> tuple[str, ...]: + return tuple(str(_stream_item_field(chunk, attr)) for attr in _RESPONSES_DELTA_FIELD_ATTRS) + + +def responses_delta_field_texts(all_chunks: Sequence[object]) -> tuple[str, ...]: + """Text each field of a ``/v1/responses`` turn spelled out in its delta events. + + One field's deltas join as they streamed, since a finding can be split across them, and + separate fields stay apart, so a reasoning summary running into the visible answer cannot + spell out text that neither of them carries on its own. + """ + deltas: Final = tuple( + (_responses_delta_field(chunk), delta) + for chunk in all_chunks + if _stream_item_field(chunk, "type") in _RESPONSES_DELTA_EVENT_TYPES + and isinstance(delta := _stream_item_field(chunk, "delta"), str) + ) + return tuple( + "".join(delta for field, delta in deltas if field == streamed_field) + for streamed_field in dict.fromkeys(field for field, _ in deltas) + ) + + +def responses_deltas_absent_from_body(all_chunks: Sequence[object], body: ResponsesAPIResponse) -> tuple[str, ...]: + """Streamed delta text the terminal body does not account for. + + Reasoning summaries and tool-call arguments reach the client through delta events that some + providers never repeat in the finished body, so a scan of that body alone would not have seen + them. Comparison runs over the JSON-escaped body so text carrying quotes or newlines is not + reported missing purely because of escaping. + """ + encoded_body: Final = json.dumps(body.model_dump(mode="json"), default=str) + return tuple( + text for text in responses_delta_field_texts(all_chunks) if text and json.dumps(text)[1:-1] not in encoded_body + ) + + +def is_terminal_error_stream(all_chunks: Sequence[object]) -> bool: + """Whether the buffered stream is only the refusal an earlier guardrail in the chain emitted. + + `post_call` guardrails are composed, so a hook can be handed the terminal error items a + preceding one produced. Those carry no content to scan, and replacing them would hide the + refusal the client is owed. + """ + if all_chunks and all(_stream_item_type(chunk) == "error" for chunk in all_chunks): + return True + return is_sse_error_stream(all_chunks) diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 46026c12d24..c58b044c0bf 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -59,6 +59,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.repelloai import ( from litellm.types.proxy.guardrails.guardrail_hooks.singulr import ( SingulrGuardrailConfigModel, ) +from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrailConfigModel, +) from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import ( ToolPermissionGuardrailConfigModel, ) @@ -140,6 +143,7 @@ class SupportedGuardrailIntegrations(Enum): REPELLOAI = "repelloai" SINGULR = "singulr" HEADROOM = "headroom" + THIRDLAW = "thirdlaw" COMPRESR = "compresr" TYPESAFE = "typesafe" STRAIKER = "straiker" @@ -1201,6 +1205,7 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o HiddenlayerGuardrailConfigModel, QostodianNexusConfigModel, VigilGuardGuardrailConfigModel, + ThirdlawGuardrailConfigModel, SingulrGuardrailConfigModel, Agent365GuardrailConfigModel, ): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py new file mode 100644 index 00000000000..12dfb8b3833 --- /dev/null +++ b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py @@ -0,0 +1,130 @@ +from collections.abc import Mapping, Sequence +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel + + +class ThirdlawGuardrailRequestMetadata(BaseModel): + """LiteLLM context for one guardrail evaluation, sent to the ThirdLaw intervene-service.""" + + model_config = ConfigDict(frozen=True, extra="allow") + + litellm_version: str + litellm_call_id: str | None = None + litellm_trace_id: str | None = None + model: str | None = None + user_api_key_hash: str | None = None + user_api_key_alias: str | None = None + user_api_key_user_id: str | None = None + user_api_key_user_email: str | None = None + user_api_key_team_id: str | None = None + user_api_key_team_alias: str | None = None + user_api_key_end_user_id: str | None = None + user_api_key_org_id: str | None = None + user_api_key_project_id: str | None = None + user_api_key_project_alias: str | None = None + user_api_key_org_alias: str | None = None + agent_id: str | None = None + litellm_session_id: str | None = None + tags: tuple[str, ...] | None = None + user_api_key_auth_metadata: Mapping[str, object] | None = None + project_metadata: Mapping[str, object] | None = None + organization_metadata: Mapping[str, object] | None = None + + +class ThirdlawGuardrailRequest(BaseModel): + """POST body for the intervene-service full-payload guardrail endpoint (/guardrails/litellm/v2).""" + + model_config = ConfigDict(frozen=True) + + event_type: Literal["pre_call", "during_call", "post_call"] + metadata: ThirdlawGuardrailRequestMetadata + request_url: str | None = None + request_headers: Mapping[str, str] | None = None + request_body: Mapping[str, object] | None = None + response_body: Mapping[str, object] | None = None + response_headers: Mapping[str, str] | None = None + # The buffered stream beside the assembled body: the events a /v1/responses or chat completions + # client received, or the raw SSE text a /v1/messages client received. The service may fold + # either side and fall back to the other, without a LiteLLM release in between. + response_chunks: Sequence[Mapping[str, object]] | None = None + response_sse: str | None = None + # Text a /v1/responses stream delivered in delta events that its terminal body does not repeat + # (reasoning summaries, tool-call arguments on some providers). Present only when non-empty. + streamed_deltas_not_in_body: Sequence[str] | None = None + additional_provider_specific_params: Mapping[str, object] | None = None + + +class ThirdlawGuardrailResponse(BaseModel): + """Decision returned by the intervene-service. + + ``request_body`` / ``response_body`` are full replacements with shallow top-level + overlay semantics: a top-level key that is present is authoritative and must be + complete (e.g. a replaced ``choices`` list must carry every choice); omitted + top-level keys keep their original values. + """ + + model_config = ConfigDict(frozen=True, extra="allow") + + action: Literal["allow", "block", "modify_request", "modify_response"] + request_body: Mapping[str, object] | None = None + response_body: Mapping[str, object] | None = None + response_status: int | None = None + message: str = "" + + +class ThirdlawGuardrailConfigModelOptionalParams(BaseModel): + additional_headers: str | None = Field( + default=None, + description="Comma-separated list of inbound request header names whose RAW values ThirdLaw should receive. Every inbound header is always forwarded; credential headers (authorization, x-api-key, cookie) are forwarded as ***REDACTED*** unless listed here. Example: authorization,x-request-id.", + ) + unreachable_fallback: Literal["fail_closed", "fail_open"] = Field( + default="fail_closed", + description="Controls LiteLLM behavior when ThirdLaw is unreachable (network error, timeout, 502-504). fail_closed blocks the request. fail_open allows the request to continue. A block decision from ThirdLaw always blocks regardless of this setting.", + ) + unscannable_stream_fallback: Literal["fail_closed", "fail_open"] = Field( + default="fail_closed", + description="Controls LiteLLM behavior when a streamed response cannot be assembled into a scannable shape (for example /v1/responses and text-completion streams). fail_closed refuses the stream. fail_open forwards it unscanned, which lets a caller pick such an endpoint to bypass response moderation.", + ) + send_stream_chunks: bool | None = Field( + default=False, + description="If true, a finished streamed response is posted both as the assembled provider body in response_body and as the buffered stream beside it: response_chunks for /v1/responses and chat completions, response_sse for /v1/messages. Set to false to post the assembled body alone.", + ) + streaming_buffer_until_moderated: bool | None = Field( + default=True, + description="If true (default), a streamed response is withheld until ThirdLaw has moderated the assembled response; a modify_response decision is then applied before anything reaches the client.", + ) + streaming_end_of_stream_only: bool | None = Field( + default=True, + description="If true (default), ThirdLaw is only called once, when a streamed response finishes, instead of periodically during streaming. Set to false to enable interim block-only checks via streaming_sampling_rate.", + ) + streaming_sampling_rate: int | None = Field( + default=5, + description="When streaming_end_of_stream_only is false, check every Nth streamed chunk (in addition to the final end-of-stream check). Interim checks can only block, not modify. Ignored when streaming_end_of_stream_only is true.", + ) + api_key: str | None = Field( + default=None, + description="API key for ThirdLaw, sent as a bearer token. Env: THIRDLAW_API_KEY.", + ) + + +class ThirdlawGuardrailConfigModel(GuardrailConfigModel[ThirdlawGuardrailConfigModelOptionalParams]): + api_base: str | None = Field( + default=None, + description="ThirdLaw intervene-service base URL; /guardrails/litellm/v2 is appended unless already present. Env: THIRDLAW_API_BASE.", + json_schema_extra={ # mutable-ok: pydantic stores schema extras as plain JSON containers + "examples": [ # mutable-ok: pydantic stores schema extras as plain JSON containers + "https://api.thirdlaw.", + ] + }, + ) + guardrail_timeout: int | None = Field( + default=60, + description="Timeout for the ThirdLaw API request. In seconds.", + ) + + @staticmethod + def ui_friendly_name() -> str: + return "ThirdLaw" diff --git a/tests/documentation_tests/test_env_keys.py b/tests/documentation_tests/test_env_keys.py index 0713abc86d0..52c3a29e465 100644 --- a/tests/documentation_tests/test_env_keys.py +++ b/tests/documentation_tests/test_env_keys.py @@ -58,8 +58,20 @@ EXCLUDED_TERMINAL_VARS = { "ALACRITTY_SOCKET", } +# Guardrail integrations whose settings docs live in the separate +# BerriAI/litellm-docs repository, so they cannot be added in the same commit +# that introduces the integration here. Drop the entry once the docs land. +EXCLUDED_PENDING_DOCS_VARS = { + "THIRDLAW_API_BASE", + "THIRDLAW_API_KEY", +} + EXCLUDED_KEYS = frozenset( - EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS | EXCLUDED_INTERNAL_TUNING_VARS + EXCLUDED_TERMINAL_VARS + | EXCLUDED_GUARD_ONLY_VARS + | EXCLUDED_ROLLOUT_FLAGS + | EXCLUDED_INTERNAL_TUNING_VARS + | EXCLUDED_PENDING_DOCS_VARS ) # Directories to skip (dependencies, venvs, caches) - only scan litellm source diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py new file mode 100644 index 00000000000..ab325ccafef --- /dev/null +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -0,0 +1,2268 @@ +import json +from collections.abc import AsyncIterator, Sequence +from typing import TypeAlias +from unittest.mock import AsyncMock + +import httpx +import pytest + +from litellm.caching import DualCache +from litellm.exceptions import GuardrailRaisedException +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrail, + guardrail_class_registry, + guardrail_initializer_registry, + initialize_guardrail, +) +from litellm.proxy.guardrails.guardrail_hooks.thirdlaw.thirdlaw import ( + _BODY_STRIP_KEYS, + ThirdlawGuardrailMissingConfig, +) +from litellm.types.guardrails import ( + GuardrailEventHooks, + LitellmParams, + SupportedGuardrailIntegrations, +) +from litellm.types.llms.openai import ( + OutputTextDeltaEvent, + ReasoningSummaryTextDeltaEvent, + ResponseCompletedEvent, + ResponseCreatedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrailConfigModel, + ThirdlawGuardrailConfigModelOptionalParams, +) +from litellm.types.utils import ( + Choices, + Delta, + Message, + ModelResponse, + ModelResponseStream, + StreamingChoices, +) + +JsonDict: TypeAlias = dict[str, object] + +_API_BASE = "https://thirdlaw.test" +_ENDPOINT = "https://thirdlaw.test/guardrails/litellm/v2" + + +def _decision_response(body: JsonDict, status_code: int = 200) -> httpx.Response: + return httpx.Response( + status_code=status_code, + json=body, + request=httpx.Request("POST", _ENDPOINT), + ) + + +def _make_guardrail(*, decisions: list[httpx.Response] | None = None, **overrides: object) -> ThirdlawGuardrail: + handler = AsyncMock(spec=AsyncHTTPHandler) + if decisions is not None: + handler.post.side_effect = decisions + kwargs: dict[str, object] = { + "api_base": _API_BASE, + "api_key": "thirdlaw_secret", + "guardrail_name": "thirdlaw-guard", + "event_hook": "pre_call", + "default_on": True, + "async_handler": handler, + **overrides, + } + return ThirdlawGuardrail(**kwargs) + + +def _request_data() -> JsonDict: + body = { + "model": "gpt-5.6", + "messages": [{"role": "user", "content": "my api key is sk-user-secret"}], + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ], + "temperature": 0.2, + } + data: JsonDict = { + **body, + "litellm_call_id": "call-123", + "litellm_session_id": "session-123", + "metadata": { + "user_api_key_hash": "hash-1", + "user_api_key_alias": "alias-1", + "user_api_key_user_id": "user-1", + "user_api_key_project_id": "project-1", + "user_api_key_project_alias": "project-alias-1", + "user_api_key_org_alias": "org-alias-1", + "agent_id": "agent-1", + "tags": ["team:eng", "env:prod"], + # Mirrors litellm core's actual merge behavior: user_api_key_auth_metadata already + # carries the team's own custom metadata (here, "team_cost_center"), plus a key-level + # reserved control key ("guardrails") that litellm core's merge does not strip today. + "user_api_key_auth_metadata": { + "cost_center": "eng-42", + "team_cost_center": "team-eng-42", + "guardrails": ["leaked-key-level-guardrail"], + "opted_out_global_guardrails": ["leaked-opt-out"], + }, + "user_api_key_auth": UserAPIKeyAuth( + team_metadata={"cost_center": "team-eng-42", "guardrails": ["team-guard"], "tags": ["team-tag"]}, + project_metadata={ + "cost_center": "project-eng-42", + "policies": ["project-policy"], + # A project can route through a named provider credential; the name itself + # is proxy operational config, not admin-authored data, and must not leak. + "model_config": [{"azure": {"litellm_credentials": "prod-azure-cred"}}], + }, + organization_metadata={"cost_center": "org-eng-42", "disable_global_guardrails": True}, + ), + "guardrails": ["thirdlaw-guard"], + }, + "guardrails": ["thirdlaw-guard"], + "api_key": "sk-forwarded-provider-key", + "secret_fields": { + "raw_headers": { + "authorization": "Bearer sk-live-raw", + "x-request-id": "req-9", + } + }, + } + # The proxy strips the header used for LiteLLM auth (authorization here) from its + # sanitized header copy; the raw value survives only in secret_fields.raw_headers. + data["proxy_server_request"] = { + "url": "http://localhost:4000/v1/chat/completions", + "method": "POST", + "headers": { + "content-type": "application/json", + "x-request-id": "req-9", + }, + "body": {**body, "messages": [{"role": "user", "content": "snapshot message"}]}, + } + return data + + +def _model_response() -> ModelResponse: + return ModelResponse( + id="chatcmpl-1", + model="gpt-5.6", + choices=[ + Choices( + index=0, + finish_reason="tool_calls", + message=Message( + role="assistant", + content="calling tool", + tool_calls=[ + { + "id": "tool-1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "sf", "token": "sk-leak"}'}, + } + ], + ), + ) + ], + usage={"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + ) + + +def _responses_api_response() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp-1", + created_at=1700000000, + model="gpt-5.6", + object="response", + output=[ + { + "id": "msg-1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "the secret is sk-leak", "annotations": []}], + } + ], + parallel_tool_calls=True, + status="completed", + ) + + +def _sent_payload(guardrail: ThirdlawGuardrail) -> JsonDict: + return guardrail.async_handler.post.call_args.kwargs["json"] + + +async def _run_pre_call(guardrail: ThirdlawGuardrail, data: JsonDict) -> JsonDict: + return await guardrail.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=data, + call_type="completion", + ) + + +def test_requires_api_base(monkeypatch): + monkeypatch.delenv("THIRDLAW_API_BASE", raising=False) + with pytest.raises(ThirdlawGuardrailMissingConfig): + ThirdlawGuardrail(api_key="k") + + +def test_env_fallback(monkeypatch): + monkeypatch.setenv("THIRDLAW_API_BASE", "https://env.thirdlaw.test") + monkeypatch.setenv("THIRDLAW_API_KEY", "env_key") + g = ThirdlawGuardrail(guardrail_name="thirdlaw", event_hook="pre_call", default_on=True) + assert g.api_base == "https://env.thirdlaw.test/guardrails/litellm/v2" + assert g.http_headers["Authorization"] == "Bearer env_key" + + +def test_endpoint_path_not_doubled(): + g = _make_guardrail(api_base=f"{_API_BASE}/guardrails/litellm/v2") + assert g.api_base == _ENDPOINT + + +def test_default_supported_event_hooks(): + g = _make_guardrail() + assert g.supported_event_hooks == [ + GuardrailEventHooks.pre_call, + GuardrailEventHooks.post_call, + GuardrailEventHooks.during_call, + ] + + +def test_invalid_sampling_rate_rejected(): + with pytest.raises(ValueError, match="streaming_sampling_rate"): + _make_guardrail(streaming_sampling_rate=0) + + +def test_enum_value(): + assert SupportedGuardrailIntegrations.THIRDLAW.value == "thirdlaw" + + +def test_config_model_ui_name(): + assert ThirdlawGuardrailConfigModel.ui_friendly_name() == "ThirdLaw" + + +def test_registry_lookup_builds_a_working_guardrail(): + """The loader finds thirdlaw by name and the object it builds enforces decisions.""" + lp = LitellmParams(guardrail="thirdlaw", mode="pre_call", api_base=_API_BASE, api_key="k") + built = guardrail_initializer_registry["thirdlaw"](lp, {"guardrail_name": "thirdlaw-guard"}) + assert isinstance(built, guardrail_class_registry["thirdlaw"]) + assert built.api_base == _ENDPOINT + + +def test_config_driven_initialization_creates_callback(): + lp = LitellmParams(guardrail="thirdlaw", mode="pre_call", api_base=_API_BASE, api_key="k") + cb = initialize_guardrail(lp, {"guardrail_name": "thirdlaw-guard"}) + assert isinstance(cb, ThirdlawGuardrail) + assert cb.api_base == _ENDPOINT + assert cb.unreachable_fallback == "fail_closed" + assert cb.guardrail_timeout == httpx.Timeout(timeout=60, connect=5.0) + + +def test_config_driven_initialization_propagates_streaming_overrides(): + lp = LitellmParams( + guardrail="thirdlaw", + mode="pre_call", + api_base=_API_BASE, + api_key="k", + streaming_end_of_stream_only=False, + streaming_sampling_rate=10, + streaming_buffer_until_moderated=False, + ) + cb = initialize_guardrail(lp, {"guardrail_name": "thirdlaw-guard"}) + assert cb.streaming_end_of_stream_only is False + assert cb.streaming_sampling_rate == 10 + assert cb.streaming_buffer_until_moderated is False + + +def test_config_model_streaming_defaults(): + params = ThirdlawGuardrailConfigModelOptionalParams() + assert params.streaming_end_of_stream_only is True + assert params.streaming_buffer_until_moderated is True + assert params.streaming_sampling_rate == 5 + + +async def test_pre_call_payload_shape(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + out = await _run_pre_call(g, data) + + assert out["messages"] == data["messages"] + call_kwargs = g.async_handler.post.call_args.kwargs + assert call_kwargs["url"] == _ENDPOINT + assert call_kwargs["headers"]["Authorization"] == "Bearer thirdlaw_secret" + + payload = call_kwargs["json"] + assert payload["event_type"] == "pre_call" + assert payload["request_url"] == "http://localhost:4000/v1/chat/completions" + assert payload["metadata"]["user_api_key_hash"] == "hash-1" + assert payload["metadata"]["litellm_call_id"] == "call-123" + assert payload["metadata"]["model"] == "gpt-5.6" + assert payload["request_body"]["model"] == "gpt-5.6" + assert payload["request_body"]["tools"][0]["function"]["name"] == "get_weather" + assert payload["request_body"]["temperature"] == 0.2 + for stripped_key in ("secret_fields", "api_key", "metadata", "guardrails", "litellm_call_id", "litellm_session_id"): + assert stripped_key not in payload["request_body"] + assert "response_body" not in payload + assert "sk-forwarded-provider-key" not in json.dumps(payload) + + +_PROVIDER_CREDENTIALS: JsonDict = { + "api_key": "sk-forwarded-provider-key", + "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "aws_session_token": "FwoGZXIvYXdzEJr//////////wEaDEXAMPLE", + "azure_ad_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.azure-ad", + "azure_password": "hunter2-azure", + "client_secret": "oauth-client-secret-value", + "vertex_credentials": {"type": "service_account", "private_key": "-----BEGIN PRIVATE KEY-----"}, + "watsonx_region_name": "eu-de", + "extra_headers": {"authorization": "Bearer sk-provider-side"}, +} + + +async def test_no_provider_credential_reaches_thirdlaw(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = {**_request_data(), **_PROVIDER_CREDENTIALS} + await _run_pre_call(g, data) + payload = _sent_payload(g) + for key in _PROVIDER_CREDENTIALS: + assert key not in payload["request_body"], key + serialized = json.dumps(payload) + for secret in ( + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "FwoGZXIvYXdzEJr//////////wEaDEXAMPLE", + "azure-ad", + "hunter2-azure", + "oauth-client-secret-value", + "BEGIN PRIVATE KEY", + "sk-provider-side", + "sk-forwarded-provider-key", + ): + assert secret not in serialized, secret + assert payload["request_body"]["messages"] == data["messages"] + + +def test_every_credential_param_litellm_declares_is_withheld(): + """The strip set is derived from CredentialLiteLLMParams, so a provider credential + added upstream is withheld without an edit here. Guards against it being re-frozen + into a hand-maintained literal that silently falls behind.""" + from litellm.types.router import CredentialLiteLLMParams + + assert set(CredentialLiteLLMParams.model_fields) <= set(_BODY_STRIP_KEYS) + + +async def test_modify_request_cannot_restore_protected_root_controls(): + """add_litellm_data_to_request strips these from caller input; the guardrail decision lands + after that strip, so a compromised or hostile service must not be able to put them back.""" + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_request", + "request_body": { + "messages": [{"role": "user", "content": "scrubbed"}], + "max_agentic_loops": 500, + "_code_interpreter_interception_active": True, + "_code_interpreter_interception_sandbox_key": "sk-sandbox", + "mock_response": "free lunch", + "callbacks": ["attacker_callback"], + }, + } + ) + ] + ) + out = await _run_pre_call(g, _request_data()) + assert out["messages"] == [{"role": "user", "content": "scrubbed"}] + for key in ( + "max_agentic_loops", + "_code_interpreter_interception_active", + "_code_interpreter_interception_sandbox_key", + "mock_response", + "callbacks", + ): + assert key not in out, key + + +async def test_modify_request_cannot_redirect_routing_or_credentials(): + """custom_llm_provider and litellm_credential_name pick which provider and which stored + credential the call runs against, both authorized before this hook ever runs.""" + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_request", + "request_body": { + "custom_llm_provider": "attacker_provider", + "litellm_credential_name": "someone-elses-credential", + }, + } + ) + ] + ) + data = _request_data() + out = await _run_pre_call(g, data) + assert "custom_llm_provider" not in out + assert "litellm_credential_name" not in out + assert out["model"] == data["model"] + + +async def test_modify_response_cannot_rename_a_non_streaming_response_id(): + """litellm hands out an encrypted response id carrying the deployment the turn routed to, and + the client chains the next turn off it, so no response shape may have it rewritten.""" + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": {"id": "resp_attacker_supplied", "choices": []}, + } + ) + ] + ) + response = _model_response() + original_id = response.id + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out.id == original_id + + +async def test_modify_request_cannot_inject_a_provider_credential(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_request", + "request_body": { + "messages": [{"role": "user", "content": "scrubbed"}], + "aws_secret_access_key": "attacker-supplied", + "azure_ad_token": "attacker-supplied", + "vertex_credentials": {"private_key": "attacker-supplied"}, + }, + } + ) + ] + ) + out = await _run_pre_call(g, _request_data()) + assert out["messages"] == [{"role": "user", "content": "scrubbed"}] + for key in ("aws_secret_access_key", "azure_ad_token", "vertex_credentials"): + assert key not in out, key + + +async def test_pre_call_sends_live_body_not_snapshot(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, _request_data()) + messages = _sent_payload(g)["request_body"]["messages"] + assert messages == [{"role": "user", "content": "my api key is sk-user-secret"}] + + +async def test_pre_call_payload_includes_extended_identity_and_hierarchy_metadata(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, _request_data()) + metadata = _sent_payload(g)["metadata"] + + assert metadata["litellm_session_id"] == "session-123" + assert metadata["user_api_key_project_id"] == "project-1" + assert metadata["user_api_key_project_alias"] == "project-alias-1" + assert metadata["user_api_key_org_alias"] == "org-alias-1" + assert metadata["agent_id"] == "agent-1" + assert metadata["tags"] == ["team:eng", "env:prod"] + assert metadata["user_api_key_auth_metadata"] == {"cost_center": "eng-42", "team_cost_center": "team-eng-42"} + + +async def test_pre_call_payload_falls_back_to_metadata_session_id(): + """A caller can declare a session purely via `metadata.session_id` (litellm's own + + documented alternative to the `x-litellm-session-id` header, see the "missing_session_id" + reject-policy error message in litellm_pre_call_utils.py) without ever populating the + call-level `litellm_session_id` field this guardrail also reads. + """ + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + del data["litellm_session_id"] + data["metadata"]["session_id"] = "client-declared-session" + await _run_pre_call(g, data) + metadata = _sent_payload(g)["metadata"] + + assert metadata["litellm_session_id"] == "client-declared-session" + + +async def test_pre_call_payload_strips_reserved_keys_from_hierarchy_metadata(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, _request_data()) + metadata = _sent_payload(g)["metadata"] + + assert metadata["project_metadata"] == {"cost_center": "project-eng-42"} + assert metadata["organization_metadata"] == {"cost_center": "org-eng-42"} + for reserved_key in ( + "guardrails", + "tags", + "policies", + "disable_global_guardrails", + "opted_out_global_guardrails", + "model_config", + ): + assert reserved_key not in metadata["user_api_key_auth_metadata"] + assert reserved_key not in metadata["project_metadata"] + assert reserved_key not in metadata["organization_metadata"] + + +async def test_pre_call_payload_does_not_duplicate_team_metadata(): + """team_metadata is already merged into user_api_key_auth_metadata by litellm core + + (litellm_pre_call_utils.py's add_management_endpoint_metadata_to_request_metadata layers + the team's own custom metadata onto that same dict), so it must not be forwarded again + under its own key. + """ + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, _request_data()) + metadata = _sent_payload(g)["metadata"] + + assert "team_metadata" not in metadata + assert metadata["user_api_key_auth_metadata"]["team_cost_center"] == "team-eng-42" + + +async def test_pre_call_payload_omits_hierarchy_metadata_when_absent(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + data["metadata"].pop("user_api_key_auth", None) + data["metadata"].pop("tags", None) + await _run_pre_call(g, data) + metadata = _sent_payload(g)["metadata"] + + for absent_field in ("tags", "project_metadata", "organization_metadata"): + assert absent_field not in metadata + + +async def test_all_headers_forwarded_without_credentials_by_default(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, _request_data()) + headers = _sent_payload(g)["request_headers"] + assert headers["content-type"] == "application/json" + assert headers["x-request-id"] == "req-9" + assert "authorization" not in headers + assert "sk-live-raw" not in json.dumps(_sent_payload(g)) + + +async def test_additional_headers_opts_into_raw_values(): + g = _make_guardrail( + decisions=[_decision_response({"action": "allow"})], + additional_headers="Authorization , x-missing", + ) + await _run_pre_call(g, _request_data()) + headers = _sent_payload(g)["request_headers"] + assert headers["authorization"] == "Bearer sk-live-raw" + assert headers["x-request-id"] == "req-9" + assert "x-missing" not in headers + + +async def test_pre_call_block_raises_with_status(): + g = _make_guardrail( + decisions=[_decision_response({"action": "block", "message": "policy violation", "response_status": 422})] + ) + with pytest.raises(GuardrailRaisedException) as exc_info: + await _run_pre_call(g, _request_data()) + assert exc_info.value.message == "policy violation" + assert exc_info.value.status_code == 422 + assert exc_info.value.blocked_content is True + + +async def test_pre_call_modify_request_applies_content_keys_only(): + modified_tools = [ + { + "type": "function", + "function": {"name": "get_weather", "description": "[sanitized]", "parameters": {"type": "object"}}, + } + ] + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_request", + "request_body": { + "messages": [{"role": "user", "content": "my api key is [REDACTED]"}], + "tools": modified_tools, + "temperature": 0.9, + "model": "attacker-model", + "guardrails": [], + "metadata": {"user_api_key_hash": "forged"}, + "stream": True, + "secret_fields": {"raw_headers": {}}, + }, + } + ) + ] + ) + data = _request_data() + out = await _run_pre_call(g, data) + + assert out["messages"] == [{"role": "user", "content": "my api key is [REDACTED]"}] + assert out["tools"] == modified_tools + assert out["temperature"] == 0.9 + assert out["model"] == "gpt-5.6" + assert out["guardrails"] == ["thirdlaw-guard"] + assert out["metadata"]["user_api_key_hash"] == "hash-1" + assert "stream" not in out + assert out["secret_fields"]["raw_headers"]["authorization"] == "Bearer sk-live-raw" + assert data["messages"] == [{"role": "user", "content": "my api key is sk-user-secret"}] + + +async def test_pre_call_records_single_guardrail_trace(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + await _run_pre_call(g, data) + traces = data["metadata"]["standard_logging_guardrail_information"] + assert len(traces) == 1 + assert traces[0]["guardrail_status"] == "success" + assert traces[0]["guardrail_response"]["action"] == "allow" + + +async def test_pre_call_block_records_intervened_trace(): + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "no"})]) + data = _request_data() + with pytest.raises(GuardrailRaisedException): + await _run_pre_call(g, data) + traces = data["metadata"]["standard_logging_guardrail_information"] + assert len(traces) == 1 + assert traces[0]["guardrail_status"] == "guardrail_intervened" + + +async def test_during_call_block_raises(): + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "denied"})]) + with pytest.raises(GuardrailRaisedException): + await g.async_moderation_hook(data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), call_type="completion") + + +async def test_during_call_modify_is_ignored(): + g = _make_guardrail( + decisions=[_decision_response({"action": "modify_request", "request_body": {"messages": [{"role": "user"}]}})] + ) + data = _request_data() + out = await g.async_moderation_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), call_type="completion") + assert out is data + assert data["messages"] == [{"role": "user", "content": "my api key is sk-user-secret"}] + assert _sent_payload(g)["event_type"] == "during_call" + + +async def test_post_call_payload_includes_response_and_prefers_snapshot_body(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + response = _model_response() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + payload = _sent_payload(g) + assert payload["event_type"] == "post_call" + assert payload["response_body"]["choices"][0]["message"]["tool_calls"][0]["function"]["name"] == "get_weather" + assert payload["request_body"]["messages"] == [{"role": "user", "content": "snapshot message"}] + + +async def test_post_call_modify_response_rewrites_content_and_tool_calls(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": { + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "calling tool [sanitized]", + "tool_calls": [ + { + "id": "tool-1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "sf", "token": "[REDACTED]"}', + }, + } + ], + }, + } + ] + }, + } + ) + ] + ) + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + assert isinstance(out, ModelResponse) + assert out.choices[0].message.content == "calling tool [sanitized]" + assert out.choices[0].message.tool_calls[0].function.arguments == '{"city": "sf", "token": "[REDACTED]"}' + assert out.id == "chatcmpl-1" + assert out.usage.total_tokens == 15 + + +async def test_post_call_modify_response_merges_dict_responses(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": {"content": [{"type": "text", "text": "[MASKED]"}]}, + } + ) + ] + ) + anthropic_response = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "the secret is sk-leak"}], + } + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=anthropic_response + ) + assert out == { + "id": "msg_1", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "[MASKED]"}], + } + + +async def test_post_call_modify_response_rewrites_responses_api_output(): + """ResponsesAPIResponse is a Pydantic model, not a ModelResponse or a raw dict.""" + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": { + "output": [ + { + "id": "msg-1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "[REDACTED]", "annotations": []}], + } + ] + }, + } + ) + ] + ) + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_responses_api_response() + ) + assert isinstance(out, ResponsesAPIResponse) + assert out.model_dump()["output"][0]["content"][0]["text"] == "[REDACTED]" + + +@pytest.mark.parametrize("response_factory", [_model_response, _responses_api_response]) +async def test_post_call_modify_response_carries_hidden_params(response_factory): + """model_validate() builds a fresh instance that starts with empty _hidden_params -- + the original's must be carried across explicitly or response-header forwarding breaks. + """ + response = response_factory() + response._hidden_params["additional_headers"] = {"x-request-id": "abc123"} + g = _make_guardrail( + decisions=[_decision_response({"action": "modify_response", "response_body": {"model": "gpt-5.6-redacted"}})] + ) + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out._hidden_params.get("additional_headers") == {"x-request-id": "abc123"} + + +@pytest.mark.parametrize("response_factory", [_model_response, _responses_api_response]) +async def test_post_call_forwards_response_headers_to_thirdlaw(response_factory): + """The provider's response headers land in _hidden_params before any post-call hook runs; + the guardrail must read them from there and post them alongside response_body.""" + response = response_factory() + response._hidden_params["additional_headers"] = {"x-request-id": "req-99", "llm_provider-x-ratelimit": "10"} + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await g.async_post_call_success_hook(data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response) + assert _sent_payload(g)["response_headers"] == { + "x-request-id": "req-99", + "llm_provider-x-ratelimit": "10", + } + + +async def test_post_call_omits_response_headers_when_absent(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + assert "response_headers" not in _sent_payload(g) + + +async def test_response_headers_hook_stashes_headers_and_returns_none(): + """async_post_call_response_headers_hook must never inject anything into the + client-facing response -- it exists purely to capture headers off the true + response object for ThirdLaw's own later use.""" + g = _make_guardrail() + response = _model_response() + response._hidden_params["additional_headers"] = {"x-request-id": "req-1"} + data = _request_data() + out = await g.async_post_call_response_headers_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is None + assert data["_thirdlaw_response_headers"] == {"x-request-id": "req-1"} + + +async def test_response_headers_hook_leaves_stash_untouched_when_absent(): + g = _make_guardrail() + data = _request_data() + await g.async_post_call_response_headers_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + assert "_thirdlaw_response_headers" not in data + + +async def test_post_call_success_hook_prefers_the_stash_over_the_response_object(): + """When both are present, the response-headers-hook stash wins -- it is the + reliable source; a direct read off `response` is the fallback.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + response = _model_response() + response._hidden_params["additional_headers"] = {"x-request-id": "from-response-object"} + data = _request_data() + data["_thirdlaw_response_headers"] = {"x-request-id": "from-stash"} + await g.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response) + assert _sent_payload(g)["response_headers"] == {"x-request-id": "from-stash"} + + +async def test_pre_call_strips_a_caller_forged_response_headers_stash(): + """Only _UNTRUSTED_ROOT_CONTROL_FIELDS is stripped from a caller's raw request body + before any hook sees `data`; our stash key isn't on that list, so a caller could + otherwise plant a fake value there themselves.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + data["_thirdlaw_response_headers"] = {"x-forged-by": "the caller"} + await _run_pre_call(g, data) + assert "_thirdlaw_response_headers" not in data + + +async def test_streaming_never_trusts_a_caller_forged_response_headers_stash(): + """A caller-forged stash entry must not survive to the wire payload, even when the + stream itself carries no real headers (e.g. another callback's bare generator, as in + test_streaming_falls_back_to_the_stash_when_response_carries_no_hidden_params).""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"}), _decision_response({"action": "allow"})]) + data = _request_data() + data["_thirdlaw_response_headers"] = {"x-forged-by": "the caller"} + await _run_pre_call(g, data) # the real proxy always runs this first + + async def bare_generator_with_no_hidden_params(): + for chunk in _stream_chunks(): + yield chunk + + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=bare_generator_with_no_hidden_params(), + request_data=data, + ) + ) + assert "response_headers" not in _sent_payload(g) + + +async def test_post_call_block_raises(): + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + with pytest.raises(GuardrailRaisedException) as exc_info: + await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + assert exc_info.value.message == "leaked secret" + + +async def test_post_call_malformed_modified_response_fails_closed(): + g = _make_guardrail( + decisions=[_decision_response({"action": "modify_response", "response_body": {"choices": "garbage"}})] + ) + with pytest.raises(GuardrailRaisedException): + await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + + +async def test_modify_request_without_a_body_leaves_the_request_alone(): + """A decision that says modify but carries nothing must not be read as "replace with empty".""" + g = _make_guardrail(decisions=[_decision_response({"action": "modify_request"})]) + data = _request_data() + out = await _run_pre_call(g, data) + assert out["messages"] == data["messages"] + assert out["tools"] == data["tools"] + + +async def test_modify_response_on_pre_call_is_ignored(): + """The response does not exist yet on pre_call, so the decision has nothing to apply to and + must not be mistaken for a request rewrite.""" + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": {"choices": []}, + "request_body": {"messages": [{"role": "user", "content": "smuggled"}]}, + } + ) + ] + ) + data = _request_data() + original_messages = list(data["messages"]) + out = await _run_pre_call(g, data) + assert out["messages"] == original_messages + + +async def test_modify_request_on_during_call_is_ignored(): + """during_call runs beside the LLM call, so the request has already gone out.""" + g = _make_guardrail( + decisions=[ + _decision_response( + {"action": "modify_request", "request_body": {"messages": [{"role": "user", "content": "late"}]}} + ) + ] + ) + data = _request_data() + out = await g.async_moderation_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), call_type="completion") + assert out["messages"] == data["messages"] + + +async def test_post_call_skips_a_response_shape_it_cannot_serialize(): + """An unscannable response type must not raise and take down the call.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + response = object() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + assert g.async_handler.post.await_count == 0 + + +async def test_modify_response_on_an_unsupported_shape_returns_the_original(): + """The scan ran and the service asked for a rewrite, but there is no way to apply it to this + shape, so the original is returned rather than a half-applied one.""" + + class _Unsupported: + def model_dump(self, *args: object, **kwargs: object) -> list[int]: + return [1, 2, 3] + + g = _make_guardrail(decisions=[_decision_response({"action": "modify_response", "response_body": {"choices": []}})]) + response = _Unsupported() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + + +async def test_a_rewrite_that_fails_validation_fails_closed(): + """A malformed rewrite must block rather than reach the client half-applied.""" + g = _make_guardrail( + decisions=[ + _decision_response({"action": "modify_response", "response_body": {"choices": [{"index": "not-an-int"}]}}) + ] + ) + with pytest.raises(GuardrailRaisedException) as exc_info: + await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + assert "malformed modified response" in exc_info.value.message + + +async def test_a_responses_rewrite_that_fails_validation_fails_closed(): + g = _make_guardrail( + decisions=[_decision_response({"action": "modify_response", "response_body": {"created_at": "nonsense"}})] + ) + with pytest.raises(GuardrailRaisedException) as exc_info: + await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=_responses_api_response() + ) + assert "malformed modified response" in exc_info.value.message + + +async def test_headers_fall_back_to_metadata_when_there_is_no_proxy_server_request(): + """Non-proxy call paths carry the inbound headers on metadata instead.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + del data["proxy_server_request"] + data["metadata"]["headers"] = {"x-demo-trace": "trace-42"} + await _run_pre_call(g, data) + assert _sent_payload(g)["request_headers"]["x-demo-trace"] == "trace-42" + + +async def test_an_unserializable_request_body_is_posted_as_absent_not_raised(): + """A body the guardrail cannot serialize must not fail live traffic; the scan still runs.""" + + class _Unserializable: + def __repr__(self) -> str: + raise RuntimeError("no repr for you") + + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + data["messages"] = _Unserializable() + out = await _run_pre_call(g, data) + assert out is data + assert "request_body" not in _sent_payload(g) + + +class _DictDumping: + """Serializes for the scan but is not a shape the rewrite knows how to rebuild.""" + + def model_dump(self, *args: object, **kwargs: object) -> JsonDict: + return {"choices": [{"message": {"content": "hi"}}]} + + +async def test_a_rewrite_for_a_shape_it_cannot_rebuild_returns_the_original(): + """The scan runs because the response serializes, but there is no constructor for this shape, + so the original is returned rather than a dict standing in for a typed response.""" + g = _make_guardrail( + decisions=[ + _decision_response( + {"action": "modify_response", "response_body": {"choices": [{"message": {"content": "nope"}}]}} + ) + ] + ) + response = _DictDumping() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + + +async def test_a_response_that_cannot_be_serialized_skips_the_scan(): + """A response whose dump raises must not take the call down with it.""" + + class _Exploding: + def model_dump(self, *args: object, **kwargs: object) -> JsonDict: + raise RuntimeError("cannot dump") + + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + response = _Exploding() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + assert g.async_handler.post.await_count == 0 + + +def test_the_guardrail_advertises_its_config_model(): + """The proxy reads the config schema off the class to render and validate the guardrail, so a + missing model silently drops every documented default.""" + model = ThirdlawGuardrail.get_config_model() + assert model is not None + assert model.ui_friendly_name() == "ThirdLaw" + optional_params_model = model.model_fields["optional_params"].annotation + assert ThirdlawGuardrailConfigModelOptionalParams in getattr(optional_params_model, "__args__", ()) + assert ThirdlawGuardrailConfigModelOptionalParams().send_stream_chunks is False + + +def _connect_error() -> httpx.ConnectError: + return httpx.ConnectError("connection refused", request=httpx.Request("POST", _ENDPOINT)) + + +async def test_unreachable_fail_closed_raises(): + g = _make_guardrail() + g.async_handler.post.side_effect = _connect_error() + with pytest.raises(GuardrailRaisedException) as exc_info: + await _run_pre_call(g, _request_data()) + assert exc_info.value.blocked_content is False + + +async def test_unreachable_fail_open_passes_through(): + g = _make_guardrail(unreachable_fallback="fail_open") + g.async_handler.post.side_effect = _connect_error() + data = _request_data() + out = await _run_pre_call(g, data) + assert out is data + traces = data["metadata"]["standard_logging_guardrail_information"] + assert traces[0]["guardrail_status"] == "guardrail_failed_to_respond" + + +async def test_http_500_fails_closed_even_with_fail_open(): + g = _make_guardrail( + unreachable_fallback="fail_open", + decisions=[_decision_response({"detail": "boom"}, status_code=500)], + ) + with pytest.raises(GuardrailRaisedException): + await _run_pre_call(g, _request_data()) + + +async def test_http_503_respects_fail_open(): + g = _make_guardrail( + unreachable_fallback="fail_open", + decisions=[_decision_response({"detail": "overloaded"}, status_code=503)], + ) + data = _request_data() + assert await _run_pre_call(g, data) is data + + +_EVALUATION_TIMEOUT_MESSAGE = ( + "ThirdLaw did not finish evaluating this request within 10 seconds, so the request was not evaluated." +) + + +def _fail_closed_detail(reason: str = "evaluation_timeout") -> JsonDict: + return { + "detail": { + "error": "evaluation_unavailable", + "reason": reason, + "message": _EVALUATION_TIMEOUT_MESSAGE, + } + } + + +def _html_error_response(status_code: int = 503) -> httpx.Response: + return httpx.Response( + status_code=status_code, + content=b"503 Service Temporarily Unavailable", + headers={"content-type": "text/html"}, + request=httpx.Request("POST", _ENDPOINT), + ) + + +async def _raised_message(g: ThirdlawGuardrail) -> str: + with pytest.raises(GuardrailRaisedException) as exc_info: + await _run_pre_call(g, _request_data()) + return exc_info.value.message + + +async def test_fail_closed_503_surfaces_the_service_reason_not_the_status_line(): + g = _make_guardrail(decisions=[_decision_response(_fail_closed_detail(), status_code=503)]) + message = await _raised_message(g) + assert _EVALUATION_TIMEOUT_MESSAGE in message + assert "reason=evaluation_timeout" in message + assert "Service Unavailable" not in message + + +async def test_fail_closed_503_records_the_service_reason_on_the_guardrail_trace(): + g = _make_guardrail(decisions=[_decision_response(_fail_closed_detail(), status_code=503)]) + data = _request_data() + with pytest.raises(GuardrailRaisedException): + await _run_pre_call(g, data) + traces = data["metadata"]["standard_logging_guardrail_information"] + assert traces[0]["guardrail_status"] == "guardrail_failed_to_respond" + assert traces[0]["guardrail_response"]["error"] == f"{_EVALUATION_TIMEOUT_MESSAGE} (reason=evaluation_timeout)" + + +async def test_non_json_503_body_falls_back_to_the_status_line(): + g = _make_guardrail() + g.async_handler.post.return_value = _html_error_response() + assert "503 Service Unavailable" in await _raised_message(g) + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + ({"error": "evaluation_unavailable"}, "503 Service Unavailable"), + ({"detail": {"error": "evaluation_unavailable"}}, "503 Service Unavailable"), + ({"detail": {"message": ""}}, "503 Service Unavailable"), + ({"detail": "policy engine is draining"}, "policy engine is draining"), + ], +) +async def test_503_detail_shapes_a_service_may_send(body: JsonDict, expected: str): + g = _make_guardrail(decisions=[_decision_response(body, status_code=503)]) + assert expected in await _raised_message(g) + + +async def test_structured_503_passes_through_under_fail_open_and_still_records_the_reason(): + g = _make_guardrail( + unreachable_fallback="fail_open", + decisions=[_decision_response(_fail_closed_detail(reason="evaluation_capacity_exceeded"), status_code=503)], + ) + data = _request_data() + assert await _run_pre_call(g, data) is data + traces = data["metadata"]["standard_logging_guardrail_information"] + assert traces[0]["guardrail_status"] == "guardrail_failed_to_respond" + assert "reason=evaluation_capacity_exceeded" in traces[0]["guardrail_response"]["error"] + + +def _stream_chunks() -> list[ModelResponseStream]: + return [ + ModelResponseStream( + id="chunk-1", + model="gpt-5.6", + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="the secret "))], + ), + ModelResponseStream( + id="chunk-1", + model="gpt-5.6", + choices=[StreamingChoices(index=0, delta=Delta(content="is sk-leak"))], + ), + ModelResponseStream( + id="chunk-1", + model="gpt-5.6", + choices=[StreamingChoices(index=0, delta=Delta(content=None), finish_reason="stop")], + ), + ] + + +async def _aiter(items: Sequence[object]) -> AsyncIterator[object]: + for item in items: + yield item + + +async def _collect(agen: AsyncIterator[object]) -> list[object]: + return [item async for item in agen] + + +class _StreamWithHiddenParams: + """Mimics CustomStreamWrapper: an async-iterable whose chunks are plain ModelResponseStream + objects, but which itself carries _hidden_params (the provider response headers, snapshotted + once at stream open).""" + + def __init__(self, items: Sequence[object], hidden_params: JsonDict) -> None: + self._items = items + self._hidden_params = hidden_params + + def __aiter__(self) -> AsyncIterator[object]: + return _aiter(self._items) + + +async def test_streaming_buffered_allow_replays_original_chunks(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + payload = _sent_payload(g) + assert payload["event_type"] == "post_call" + assert payload["response_body"]["choices"][0]["message"]["content"] == "the secret is sk-leak" + + +async def test_streaming_forwards_response_headers_from_the_stream_wrapper(): + """Individual ModelResponseStream chunks carry no _hidden_params of their own -- the headers + live on the CustomStreamWrapper (or Responses/Anthropic Messages equivalent) that the proxy + passes in as `response`, snapshotted once at stream open, so they must be read from there + rather than from the buffered chunks.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + stream = _StreamWithHiddenParams(_stream_chunks(), hidden_params={"additional_headers": {"x-request-id": "req-77"}}) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=stream, request_data=_request_data() + ) + ) + assert _sent_payload(g)["response_headers"] == {"x-request-id": "req-77"} + + +async def test_streaming_falls_back_to_the_stash_when_response_carries_no_hidden_params(): + """Simulates another guardrail running ahead of ThirdLaw in litellm.callbacks and + handing it a bare generator with no _hidden_params. The streaming hook must fall + back to the stash async_post_call_response_headers_hook left in request_data. + """ + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + + async def bare_generator_with_no_hidden_params(): + for chunk in _stream_chunks(): + yield chunk + + request_data = _request_data() + request_data["_thirdlaw_response_headers"] = {"x-request-id": "req-78"} + + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=bare_generator_with_no_hidden_params(), + request_data=request_data, + ) + ) + assert _sent_payload(g)["response_headers"] == {"x-request-id": "req-78"} + + +async def test_sampled_stream_forwards_response_headers_from_the_stream_wrapper(): + """Same as the buffered case, for the end-of-stream call on the sampled (non-buffered) path.""" + g = _make_guardrail( + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + streaming_sampling_rate=100, + decisions=[_decision_response({"action": "allow"})], + ) + stream = _StreamWithHiddenParams(_stream_chunks(), hidden_params={"additional_headers": {"x-request-id": "req-78"}}) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=stream, request_data=_request_data() + ) + ) + assert _sent_payload(g)["response_headers"] == {"x-request-id": "req-78"} + + +async def test_streaming_buffered_modify_emits_rewritten_response(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": { + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "the secret is [REDACTED]"}, + } + ] + }, + } + ) + ] + ) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + emitted_text = "".join( + choice.delta.content or "" + for chunk in out + if isinstance(chunk, ModelResponseStream) + for choice in chunk.choices + ) + assert emitted_text == "the secret is [REDACTED]" + assert "sk-leak" not in emitted_text + + +def _redacting_modify_decision() -> httpx.Response: + return _decision_response( + { + "action": "modify_response", + "response_body": { + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "the secret is [REDACTED]"}, + } + ] + }, + } + ) + + +async def test_masking_guardrail_does_not_buffer_and_replay_originals(): + """Buffered replay hands back the unredacted chunks, so a masking guardrail must not buffer.""" + g = _make_guardrail(decisions=[_redacting_modify_decision()], mask_response_content=True) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + emitted = "".join( + choice.delta.content or "" + for chunk in out + if isinstance(chunk, ModelResponseStream) + for choice in chunk.choices + ) + assert emitted == "the secret is sk-leak" + assert "[REDACTED]" not in emitted + + +async def test_streaming_buffered_block_raises_streaming_callback_error(): + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + with pytest.raises(StreamingCallbackError, match="leaked secret"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + + +async def test_streaming_buffered_holds_chunks_until_decision(): + call_order: list[str] = [] + + async def _recording_stream() -> AsyncIterator[object]: + for chunk in _stream_chunks(): + call_order.append("chunk_consumed") + yield chunk + + g = _make_guardrail() + + async def _post(*args, **kwargs): + call_order.append("guardrail_called") + return _decision_response({"action": "allow"}) + + g.async_handler.post.side_effect = _post + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_recording_stream(), request_data=_request_data() + ) + ) + assert len(out) == 3 + assert call_order == ["chunk_consumed", "chunk_consumed", "chunk_consumed", "guardrail_called"] + + +async def test_streaming_sampled_interim_block_terminates_stream(): + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail( + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + streaming_sampling_rate=1, + decisions=[_decision_response({"action": "block", "message": "bad interim"})], + ) + agen = g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + first = await agen.__anext__() + assert isinstance(first, ModelResponseStream) + with pytest.raises(StreamingCallbackError, match="bad interim"): + await agen.__anext__() + + +def _truncated_tool_use_sse_frames() -> list[bytes]: + """A tool_use block whose argument JSON was cut off by max_tokens. + + The secret only ever exists inside the unparseable fragment, so if assembly drops it the + scan never sees it while the original frames still reach the client. + """ + events = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_tool", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 9, "output_tokens": 0}, + }, + }, + ), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}, + }, + ), + ( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "input_json_delta", "partial_json": '{"query": "sk-live-TRUNC'}, + }, + ), + ( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}}, + ), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(body)}\n\n".encode() for name, body in events] + + +async def test_a_truncated_tool_argument_still_reaches_the_scan(): + """Text the model emitted inside a tool argument reaches the client in the replayed frames, + so dropping it from the assembled body would let a caller pick truncation as a bypass.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_truncated_tool_use_sse_frames()), + request_data=_request_data(), + ) + ) + assert "sk-live-TRUNC" in json.dumps(_sent_payload(g)["response_body"]) + + +async def test_a_truncated_tool_argument_can_still_be_blocked(): + """The scan seeing the text is only half the fix; the block decision has to land.""" + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "secret in tool args"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_truncated_tool_use_sse_frames()), + request_data=_request_data(), + ) + ) + blob = b"".join(item for item in out if isinstance(item, bytes)) + assert b"secret in tool args" in blob + assert b"sk-live-TRUNC" not in blob + + +def _anthropic_sse_frames() -> list[bytes]: + events = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_abc", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 9, "output_tokens": 0}, + }, + }, + ), + ( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello there"}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 4}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + +async def test_streaming_raw_sse_allow_replays_frames(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + frames = _anthropic_sse_frames() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(frames), request_data=_request_data() + ) + ) + assert out == frames + posted = _sent_payload(g)["response_body"] + assert "choices" not in posted + assert posted["type"] == "message" + assert posted["id"] == "msg_abc" + assert posted["content"] == [{"type": "text", "text": "hello there"}] + assert posted["stop_reason"] == "end_turn" + assert posted["usage"] == {"input_tokens": 9, "output_tokens": 4} + + +def _anthropic_thinking_sse_frames() -> list[bytes]: + """A turn carrying everything the chat-completions shape has no field for.""" + events = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_thinking", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 9, + "output_tokens": 0, + "cache_creation_input_tokens": 120, + "cache_read_input_tokens": 400, + }, + }, + }, + ), + ( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "thinking", "thinking": "", "signature": ""}, + }, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "weighing it"}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "signature_delta", "signature": "sig-xyz"}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "content_block_start", + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + ), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "hello there"}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "stop_sequence", "stop_sequence": "END"}, + "usage": {"output_tokens": 4}, + }, + ), + ("message_stop", {"type": "message_stop"}), + ] + return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events] + + +async def test_a_messages_stream_keeps_what_the_chat_shape_has_no_field_for(): + """Folding the stream into a ModelResponse dropped thinking blocks, the signature, + stop_sequence and the cache token split before the scan ever saw them.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_thinking_sse_frames()), + request_data=_request_data(), + ) + ) + posted = _sent_payload(g)["response_body"] + assert posted["content"][0] == {"type": "thinking", "thinking": "weighing it", "signature": "sig-xyz"} + assert posted["content"][1] == {"type": "text", "text": "hello there"} + assert posted["stop_sequence"] == "END" + assert posted["usage"]["cache_creation_input_tokens"] == 120 + assert posted["usage"]["cache_read_input_tokens"] == 400 + + +async def test_a_messages_stream_rewrite_is_re_emitted_as_anthropic_frames(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": {"content": [{"type": "text", "text": "hello [REDACTED]"}]}, + } + ) + ] + ) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_sse_frames()), + request_data=_request_data(), + ) + ) + joined = b"".join(item for item in out if isinstance(item, bytes)) + assert b"hello [REDACTED]" in joined + assert b"hello there" not in joined + + +async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed(): + """The overlay is a shallow merge, so "choices" would land beside the untouched "content" + and the re-emitted stream would carry the very text the rewrite asked to redact.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[_redacting_modify_decision()]) + with pytest.raises(StreamingCallbackError, match="answered a /v1/messages stream"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_sse_frames()), + request_data=_request_data(), + ) + ) + + +async def test_streaming_raw_sse_block_emits_anthropic_error_frame(): + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_sse_frames()), + request_data=_request_data(), + ) + ) + assert len(out) == 1 + assert isinstance(out[0], bytes) + assert b"event: error" in out[0] + assert b"leaked secret" in out[0] + + +def _unscannable_chunks() -> list[dict]: + """A stream shape with no assembler and no wire format of its own. + + Text-completion streams arrive like this: plain dicts that are neither ModelResponseStream, + raw Anthropic SSE frames, nor Responses API events, so there is no format to refuse them in. + """ + return [ + {"text": "the secret ", "index": 0}, + {"text": "is sk-leak", "index": 0}, + ] + + +def _truncated_responses_chunks() -> list[dict]: + """A /v1/responses stream that died before its terminal event. + + response.created also carries a body, but an empty one, so only a terminal event yields + something scannable. Without one the turn cannot be scanned at all. + """ + return [ + {"type": "response.output_text.delta", "delta": "the secret "}, + {"type": "response.output_text.delta", "delta": "is sk-leak"}, + ] + + +async def test_unscannable_stream_fails_closed_by_default(): + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[]) + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_unscannable_chunks()), + request_data=_request_data(), + ) + ) + assert g.async_handler.post.await_count == 0 + + +async def test_an_opaque_sse_stream_is_not_refused_in_anthropic_frames(): + """A Google :streamGenerateContent stream is raw SSE but not Anthropic, so an Anthropic + error frame would refuse it in a format its client cannot parse.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + google_frames = [b'data: {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}\n\n'] + g = _make_guardrail(decisions=[]) + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(google_frames), + request_data=_request_data(), + ) + ) + assert g.async_handler.post.await_count == 0 + + +async def test_modify_request_on_a_stream_is_ignored(): + """The request is long gone by the time a stream finishes, so a request rewrite has nothing + to apply to and the original chunks are released.""" + g = _make_guardrail( + decisions=[ + _decision_response( + {"action": "modify_request", "request_body": {"messages": [{"role": "user", "content": "late"}]}} + ) + ] + ) + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + + +async def test_a_chat_stream_rewrite_that_fails_validation_blocks_the_stream(): + """A rewrite the assembler cannot turn back into a response must not reach the client.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail( + decisions=[ + _decision_response({"action": "modify_response", "response_body": {"choices": [{"index": "not-an-int"}]}}) + ] + ) + with pytest.raises((StreamingCallbackError, GuardrailRaisedException)): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_stream_chunks()), + request_data=_request_data(), + ) + ) + + +async def test_a_messages_stream_rewrite_answered_with_choices_is_refused(): + """The Messages body carries its text in "content"; answering with "choices" means the service + replied in the wrong body shape, which would silently blank the response.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[_decision_response({"action": "modify_response", "response_body": {"choices": []}})]) + with pytest.raises(StreamingCallbackError, match="carries its text in"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_sse_frames()), + request_data=_request_data(), + ) + ) + + +async def test_an_opaque_sse_stream_posts_no_buffered_stream_beside_the_body(): + """There is no chunk shape to post for a surface with no assembler, so the stream fields stay + off the payload instead of carrying a half-understood blob.""" + google_frames = [b'data: {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}\n\n'] + g = _make_guardrail(send_stream_chunks=True, unscannable_stream_fallback="fail_open", decisions=[]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(google_frames), + request_data=_request_data(), + ) + ) + assert g.async_handler.post.await_count == 0 + + +async def test_a_sampled_stream_blocks_on_the_final_decision(): + """The interim checks passed but the assembled response did not, so the terminal decision has + to stop the stream even though chunks already went out.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail( + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + streaming_sampling_rate=100, + decisions=[_decision_response({"action": "block", "message": "final says no"})], + ) + with pytest.raises(StreamingCallbackError, match="final says no"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_stream_chunks()), + request_data=_request_data(), + ) + ) + + +async def test_a_sampled_stream_cannot_apply_a_late_modify_response(): + """Chunks were already delivered, so a rewrite arriving at end of stream is dropped with a + warning rather than silently appearing to have been applied.""" + g = _make_guardrail( + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + streaming_sampling_rate=100, + decisions=[_redacting_modify_decision()], + ) + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + + +async def test_an_earlier_guardrails_refusal_is_passed_through_not_replaced(): + """post_call guardrails compose, so this hook can be handed the terminal error frames a + preceding one emitted. Replacing them would hide the rejection the client is owed.""" + upstream_refusal = [b'event: error\ndata: {"type": "error", "error": {"message": "blocked by presidio"}}\n\n'] + g = _make_guardrail(decisions=[]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(upstream_refusal), + request_data=_request_data(), + ) + ) + assert out == upstream_refusal + assert g.async_handler.post.await_count == 0 + + +def _responses_stream_chunks(text: str = "the secret is sk-leak") -> list[object]: + """A complete /v1/responses stream, the shape that crashed stream_chunk_builder. + + The terminal response.completed event carries the finished body; every event before it + carries either an empty body or a delta. + """ + body = ResponsesAPIResponse( + id="resp_encrypted_abc", + created_at=1, + model="claude-haiku-4-5", + object="response", + output=[ + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ], + parallel_tool_calls=True, + tool_choice="auto", + tools=[], + top_p=1.0, + ) + return [ + ResponseCreatedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_CREATED, + response=ResponsesAPIResponse( + id="resp_encrypted_abc", + created_at=1, + model="claude-haiku-4-5", + object="response", + output=[], + parallel_tool_calls=True, + tool_choice="auto", + tools=[], + top_p=1.0, + ), + ), + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta=text, + ), + ResponseCompletedEvent(type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, response=body), + ] + + +async def test_a_responses_stream_is_scanned_in_the_native_responses_body_shape(): + """stream_chunk_builder raised KeyError: 'model' on these events, refusing the whole stream. + + The body posted to the service must be the Responses shape its non-streaming half already + sends, not a chat-completions body with a fabricated "choices". + """ + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + chunks = _responses_stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + assert g.async_handler.post.await_count == 1 + posted = g.async_handler.post.await_args.kwargs["json"]["response_body"] + assert "choices" not in posted + assert posted["object"] == "response" + assert posted["id"] == "resp_encrypted_abc" + assert posted["output"][0]["content"][0]["text"] == "the secret is sk-leak" + + +async def test_a_responses_stream_block_arrives_as_a_responses_error_event(): + """A raised exception reaches the client as the proxy's {"error": ...} blob, which has no + top-level "type" and so is not a Responses event at all.""" + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_responses_stream_chunks()), + request_data=_request_data(), + ) + ) + assert [getattr(item, "type", None) for item in out] == [ResponsesAPIStreamEvents.ERROR] + assert out[0].error.message == "leaked secret" + + +async def test_a_responses_stream_rewrite_arrives_as_responses_events(): + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": { + "output": [ + { + "id": "msg_1", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + {"type": "output_text", "text": "the secret is [REDACTED]", "annotations": []} + ], + } + ] + }, + } + ) + ] + ) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_responses_stream_chunks()), + request_data=_request_data(), + ) + ) + emitted = "".join(item.delta for item in out if getattr(item, "type", None) == "response.output_text.delta") + assert emitted == "the secret is [REDACTED]" + assert "sk-leak" not in emitted + completed = [item for item in out if getattr(item, "type", None) == "response.completed"] + assert len(completed) == 1 + assert completed[0].response.id == "resp_encrypted_abc" + + +async def test_a_responses_rewrite_cannot_rename_the_encrypted_response_id(): + """litellm hands out an encrypted response id and the client chains the next turn off it + with previous_response_id, so a rewrite must not be able to replace it.""" + g = _make_guardrail( + decisions=[ + _decision_response( + {"action": "modify_response", "response_body": {"id": "resp_attacker_supplied"}}, + ) + ] + ) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_responses_stream_chunks()), + request_data=_request_data(), + ) + ) + ids = {getattr(item, "response", None) and item.response.id for item in out} + assert "resp_attacker_supplied" not in ids + assert "resp_encrypted_abc" in ids + + +async def test_a_truncated_responses_stream_is_refused_as_a_responses_error_event(): + g = _make_guardrail(decisions=[]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_truncated_responses_chunks()), + request_data=_request_data(), + ) + ) + assert [getattr(item, "type", None) for item in out] == [ResponsesAPIStreamEvents.ERROR] + assert "could not be assembled for scanning" in out[0].error.message + assert g.async_handler.post.await_count == 0 + + +async def test_a_responses_error_event_continues_the_streams_sequence_numbering(): + """An error event restarting at zero after N events would arrive out of order.""" + chunks = _responses_stream_chunks() + chunks[-1].__dict__["sequence_number"] = 11 + g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out[0].sequence_number == 12 + + +async def test_delta_text_missing_from_the_terminal_body_is_posted_beside_it(): + """Reasoning summaries reach the client through deltas some providers never repeat in the + finished body. A scan of that body alone would miss them, so they ride along in their own + field and the body itself stays the shape the non-streaming route posts.""" + chunks = _responses_stream_chunks() + chunks.insert( + 2, + ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + summary_index=0, + delta="thinking about ", + ), + ) + chunks.insert( + 3, + ReasoningSummaryTextDeltaEvent( + type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, + item_id="rs_1", + output_index=0, + summary_index=0, + delta="sk-other-leak", + ), + ) + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + posted = _sent_payload(g) + assert posted["streamed_deltas_not_in_body"] == ["thinking about sk-other-leak"] + assert "choices" not in posted["response_body"] + assert posted["response_body"]["output"][0]["content"][0]["text"] == "the secret is sk-leak" + + +async def test_the_deltas_field_is_omitted_when_the_body_already_carries_every_delta(): + """Only text the body does not account for goes beside it; a clean turn posts no field at all, + so the service sees the same payload shape it did before the field existed.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_responses_stream_chunks()), + request_data=_request_data(), + ) + ) + assert "streamed_deltas_not_in_body" not in _sent_payload(g) + + +async def test_the_deltas_field_never_appears_on_a_chat_or_messages_stream(): + """Only a /v1/responses turn can leave delta text out of its body; the other surfaces fold + every delta into what they post.""" + for chunks in (_stream_chunks(), _anthropic_sse_frames()): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert "streamed_deltas_not_in_body" not in _sent_payload(g) + + +async def test_a_responses_stream_is_posted_as_chunks_beside_the_body(): + """The body is what the proxy folded; the chunks are what the client received. Both go, so the + service can fold on its own side, and a sequence number the bridge stamps outside the model + survives the trip.""" + chunks = _responses_stream_chunks() + chunks[1].__dict__["sequence_number"] = 4 + g = _make_guardrail(send_stream_chunks=True, decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + posted = _sent_payload(g) + assert [c["type"] for c in posted["response_chunks"]] == [ + "response.created", + "response.output_text.delta", + "response.completed", + ] + assert posted["response_chunks"][1]["delta"] == "the secret is sk-leak" + assert posted["response_chunks"][1]["sequence_number"] == 4 + assert "response_sse" not in posted + assert posted["response_body"]["object"] == "response" + + +async def test_a_messages_stream_is_posted_as_the_raw_sse_text_beside_the_body(): + frames = _anthropic_sse_frames() + g = _make_guardrail(send_stream_chunks=True, decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(frames), request_data=_request_data() + ) + ) + posted = _sent_payload(g) + assert posted["response_sse"] == b"".join(frames).decode() + assert "response_chunks" not in posted + assert posted["response_body"]["type"] == "message" + + +async def test_a_chat_stream_is_posted_as_chunks_beside_the_body(): + chunks = _stream_chunks() + g = _make_guardrail(send_stream_chunks=True, decisions=[_decision_response({"action": "allow"})]) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + posted = _sent_payload(g) + assert len(posted["response_chunks"]) == len(chunks) + assert posted["response_chunks"][0]["choices"][0]["delta"]["content"] == "the secret " + assert "response_sse" not in posted + assert posted["response_body"]["choices"][0]["message"]["content"] == "the secret is sk-leak" + + +async def test_the_stream_is_not_posted_when_send_stream_chunks_is_off(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})], send_stream_chunks=False) + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_responses_stream_chunks()), + request_data=_request_data(), + ) + ) + posted = _sent_payload(g) + assert "response_chunks" not in posted + assert "response_sse" not in posted + assert posted["response_body"]["object"] == "response" + + +async def test_only_the_final_scan_carries_the_stream_not_the_interim_ones(): + """An interim scan sees a partial stream with no terminal event, which folds to nothing on the + service side, so the chunks ride only on the end-of-stream post.""" + g = _make_guardrail( + send_stream_chunks=True, + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + streaming_sampling_rate=1, + decisions=[_decision_response({"action": "allow"})] * 6, + ) + chunks = _stream_chunks() + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + posts = [call.kwargs["json"] for call in g.async_handler.post.call_args_list] + assert len(posts) == len(chunks) + 1 + assert all("response_chunks" not in post for post in posts[:-1]) + assert len(posts[-1]["response_chunks"]) == len(chunks) + + +def test_the_initializer_sends_stream_chunks_only_on_an_explicit_true(): + """An operator who never sets send_stream_chunks must not have the whole buffered stream + posted out: the config schema documents the default as False, so an omitted value is off.""" + omitted = initialize_guardrail( + LitellmParams(guardrail="thirdlaw", mode="post_call", api_base=_API_BASE, api_key="k"), + {"guardrail_name": "thirdlaw-omitted"}, + ) + explicit_off = initialize_guardrail( + LitellmParams( + guardrail="thirdlaw", mode="post_call", api_base=_API_BASE, api_key="k", send_stream_chunks=False + ), + {"guardrail_name": "thirdlaw-off"}, + ) + explicit_on = initialize_guardrail( + LitellmParams(guardrail="thirdlaw", mode="post_call", api_base=_API_BASE, api_key="k", send_stream_chunks=True), + {"guardrail_name": "thirdlaw-on"}, + ) + assert omitted.send_stream_chunks is False + assert explicit_off.send_stream_chunks is False + assert explicit_on.send_stream_chunks is True + + +@pytest.mark.parametrize("typo", ["fail_close", "failopen", "FAIL_OPEN", ""]) +async def test_unscannable_stream_fails_closed_on_a_mistyped_fallback(typo): + """Literal is not enforced at runtime, so anything but the exact opt-in must block.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[], unscannable_stream_fallback=typo) + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_unscannable_chunks()), + request_data=_request_data(), + ) + ) + + +async def test_unscannable_stream_passes_through_when_opted_into_fail_open(): + chunks = _unscannable_chunks() + g = _make_guardrail(decisions=[], unscannable_stream_fallback="fail_open") + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + assert g.async_handler.post.await_count == 0 + + +@pytest.mark.parametrize( + ("service_status", "expected_status"), + [(200, 400), (204, 400), (302, 400), (403, 403), (451, 451), (503, 503), (None, 400)], +) +async def test_block_never_travels_as_a_success_status(service_status, expected_status): + """A block is a refusal, so a success status from the service must not reach the caller.""" + body: JsonDict = {"action": "block", "message": "nope"} + if service_status is not None: + body["response_status"] = service_status + g = _make_guardrail(decisions=[_decision_response(body)]) + with pytest.raises(GuardrailRaisedException) as excinfo: + await g.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth(), + cache=DualCache(), + data=_request_data(), + call_type="completion", + ) + assert excinfo.value.status_code == expected_status diff --git a/tests/unit/proxy/guardrails/test_anthropic_sse.py b/tests/unit/proxy/guardrails/test_anthropic_sse.py new file mode 100644 index 00000000000..9e953fbfc13 --- /dev/null +++ b/tests/unit/proxy/guardrails/test_anthropic_sse.py @@ -0,0 +1,106 @@ +"""Assembly contract for raw Anthropic SSE streams. + +Four guardrails (bedrock, model_armor, tool_permission, thirdlaw) and the OTel request/response +recorder all scan the body this module assembles, so anything it drops is invisible to every one +of them. +""" + +import json +from collections.abc import Sequence + +from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_body + + +def _frames(*events: tuple[str, dict[str, object]]) -> list[bytes]: + return [f"event: {name}\ndata: {json.dumps(body)}\n\n".encode() for name, body in events] + + +def _message_start() -> tuple[str, dict[str, object]]: + return ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 0}, + }, + }, + ) + + +def _tool_block_start(index: int = 0) -> tuple[str, dict[str, object]]: + return ( + "content_block_start", + { + "type": "content_block_start", + "index": index, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}}, + }, + ) + + +def _input_delta(partial_json: str, index: int = 0) -> tuple[str, dict[str, object]]: + return ( + "content_block_delta", + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "input_json_delta", "partial_json": partial_json}, + }, + ) + + +def _assembled_tool_input(*events: tuple[str, dict[str, object]]) -> object: + """The tool block as it lands in the native Anthropic body, which is the shape the /v1/messages + scan is handed. The ModelResponse conversion keeps the raw partial JSON by another route, so + asserting through that one would pass either way.""" + body = assemble_anthropic_sse_body(_frames(*events)) + assert body is not None + content = body["content"] + assert isinstance(content, Sequence) + return content[0]["input"] + + +def test_complete_tool_arguments_assemble_as_the_object_the_model_sent(): + arguments = _assembled_tool_input( + _message_start(), + _tool_block_start(), + _input_delta('{"query": "weather'), + _input_delta(' in sf"}'), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_stop", {"type": "message_stop"}), + ) + assert arguments == {"query": "weather in sf"} + + +def test_truncated_tool_arguments_keep_the_text_instead_of_dropping_it(): + """max_tokens can cut a tool argument mid-JSON. That fragment still reached the client, so + discarding it here would hand a caller a way to smuggle text past every scanner above.""" + arguments = _assembled_tool_input( + _message_start(), + _tool_block_start(), + _input_delta('{"query": "sk-live-TRUNC'), + ( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}}, + ), + ("message_stop", {"type": "message_stop"}), + ) + assert "sk-live-TRUNC" in json.dumps(arguments) + + +def test_a_tool_block_with_no_arguments_assembles_as_an_empty_object(): + """A tool called with no arguments is not truncation, so it must not pick up a raw fragment.""" + arguments = _assembled_tool_input( + _message_start(), + _tool_block_start(), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("message_stop", {"type": "message_stop"}), + ) + assert arguments == {} diff --git a/tests/unit/proxy/guardrails/test_stream_surface.py b/tests/unit/proxy/guardrails/test_stream_surface.py new file mode 100644 index 00000000000..049b9708db5 --- /dev/null +++ b/tests/unit/proxy/guardrails/test_stream_surface.py @@ -0,0 +1,122 @@ +import json + +import pytest + +from litellm.proxy.guardrails.stream_surface import ( + StreamSurface, + classify_stream, + is_terminal_error_stream, +) +from litellm.types.llms.openai import ( + ErrorEvent, + ErrorEventError, + OutputTextDeltaEvent, + ResponseCompletedEvent, + ResponsesAPIResponse, + ResponsesAPIStreamEvents, +) +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + +def _chat_chunks() -> tuple[ModelResponseStream, ...]: + return ( + ModelResponseStream( + model="gpt-4o-mini", + choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="hi"))], + ), + ) + + +def _anthropic_sse_frames() -> tuple[bytes, ...]: + message_start = { + "type": "message_start", + "message": {"id": "msg_1", "model": "claude-haiku-4-5", "content": [], "usage": {"input_tokens": 3}}, + } + delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}} + return ( + f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode(), + f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode(), + ) + + +def _google_sse_frames() -> tuple[bytes, ...]: + """The :streamGenerateContent route marks its stream raw too, but is not Anthropic.""" + payload = {"candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}]} + return (f"data: {json.dumps(payload)}\n\n".encode(),) + + +def _responses_events() -> tuple[object, ...]: + return ( + OutputTextDeltaEvent( + type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, + item_id="msg_1", + output_index=0, + content_index=0, + delta="hi", + ), + ResponseCompletedEvent( + type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED, + response=ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-haiku-4-5", + object="response", + output=[], + parallel_tool_calls=True, + tool_choice="auto", + tools=[], + top_p=1.0, + ), + ), + ) + + +@pytest.mark.parametrize( + "chunks, expected", + [ + (_chat_chunks(), StreamSurface.CHAT_COMPLETIONS), + (_anthropic_sse_frames(), StreamSurface.ANTHROPIC_MESSAGES), + (_responses_events(), StreamSurface.RESPONSES), + (_google_sse_frames(), StreamSurface.OPAQUE_SSE), + ((), StreamSurface.CHAT_COMPLETIONS), + ], +) +def test_classify_stream_names_each_wire_format(chunks, expected): + assert classify_stream(chunks) is expected + + +def test_opaque_sse_is_not_mistaken_for_anthropic(): + """Reading a Google stream as Anthropic would refuse it in a format its client cannot parse.""" + assert classify_stream(_google_sse_frames()) is not StreamSurface.ANTHROPIC_MESSAGES + + +def test_responses_events_are_classified_from_a_plain_dict_too(): + """Some producers hand the hook already-serialized events rather than the pydantic models.""" + assert classify_stream(({"type": "response.output_text.delta", "delta": "hi"},)) is StreamSurface.RESPONSES + + +def test_a_responses_error_only_stream_is_a_terminal_error_stream(): + events = ( + ErrorEvent( + type=ResponsesAPIStreamEvents.ERROR, + sequence_number=1, + error=ErrorEventError(type="guardrail_error", code="400", message="blocked upstream"), + ), + ) + assert is_terminal_error_stream(events) is True + + +def test_an_anthropic_error_only_stream_is_a_terminal_error_stream(): + frames = (b'event: error\ndata: {"type": "error", "error": {"message": "blocked upstream"}}\n\n',) + assert is_terminal_error_stream(frames) is True + + +def test_a_stream_carrying_content_is_not_a_terminal_error_stream(): + assert is_terminal_error_stream(_responses_events()) is False + assert is_terminal_error_stream(_chat_chunks()) is False + assert is_terminal_error_stream(_anthropic_sse_frames()) is False + + +def test_an_empty_stream_is_not_a_terminal_error_stream(): + """all() over nothing is True, so an empty stream would otherwise read as somebody's refusal.""" + assert is_terminal_error_stream(()) is False diff --git a/ui/litellm-dashboard/public/assets/logos/thirdlaw.svg b/ui/litellm-dashboard/public/assets/logos/thirdlaw.svg new file mode 100644 index 00000000000..751775399ba --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/thirdlaw.svg @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts index 10ca58294b9..f81ade7571b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_configs.ts @@ -282,6 +282,12 @@ export const GUARDRAIL_PRESETS: Record = { mode: "pre_call", defaultOn: false, }, + thirdlaw: { + provider: "ThirdLaw", + guardrailNameSuggestion: "ThirdLaw Guardrail", + mode: "pre_call", + defaultOn: false, + }, promptguard: { provider: "Promptguard", guardrailNameSuggestion: "PromptGuard", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts index a27dd344c95..53474a22e83 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.test.ts @@ -27,6 +27,7 @@ const EXPECTED_PARTNER_LOGO_FILES: Record = { deepkeep: "deepkeep.svg", repelloai: "repelloai.png", straiker: "straiker.svg", + thirdlaw: "thirdlaw.svg", alice: "alice.svg", agent_365: "microsoft_azure.svg", conduct: "conduct.png", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts index d88a333d6f1..0b29ee92e31 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_garden_data.ts @@ -454,6 +454,16 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [ tags: ["Security", "Policy", "Prompt Injection"], providerKey: "Repelloai", }, + { + id: "thirdlaw", + name: "ThirdLaw", + description: + "Runtime policy enforcement for LiteLLM traffic. Evaluates requests and responses against centrally managed policies called Laws, then returns a decision to continue, modify, or block.", + category: "partner", + logo: guardrailLogoMap["ThirdLaw"], + tags: ["Policy Enforcement", "Runtime Security", "Intervention"], + providerKey: "ThirdLaw", + }, { id: "straiker", name: "Straiker", diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx index c5e07fe9624..42aebb51383 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx @@ -209,6 +209,20 @@ describe("guardrail_info_helpers", () => { expect(result.displayName).toBe("RepelloAI Argus"); expect(result.logo).toContain("repelloai.png"); }); + + it("should return thirdlaw logo when ui_friendly_name is ThirdLaw", () => { + populateGuardrailProviders({ + thirdlaw: { ui_friendly_name: "ThirdLaw" }, + }); + populateGuardrailProviderMap({ + thirdlaw: { ui_friendly_name: "ThirdLaw" }, + }); + + const result = getGuardrailLogoAndName("thirdlaw"); + + expect(result.displayName).toBe("ThirdLaw"); + expect(result.logo).toContain("thirdlaw.svg"); + }); }); describe("formatGuardrailMode", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index 8df7dfb1403..3285346bd8f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -26,6 +26,7 @@ import qohashLogo from "../../../../../public/assets/logos/qohash.jpg"; import repelloAiLogo from "../../../../../public/assets/logos/repelloai.png"; import straikerLogo from "../../../../../public/assets/logos/straiker.svg"; import xecguardLogo from "../../../../../public/assets/logos/xecguard.svg"; +import thirdlawLogo from "../../../../../public/assets/logos/thirdlaw.svg"; import zscalerLogo from "../../../../../public/assets/logos/zscaler.svg"; // Legacy enum - keeping for backward compatibility @@ -85,6 +86,7 @@ export const guardrail_provider_map: Record = { Deepkeep: "deepkeep", QostodianNexus: "qostodian_nexus", Repelloai: "repelloai", + ThirdLaw: "thirdlaw", Alice: "alice", Conduct: "conduct", }; @@ -178,6 +180,7 @@ export const shouldRenderLLMJudgeFields = (provider: string | null) => { }; export const guardrailLogoMap = { + ThirdLaw: thirdlawLogo.src, "Zscaler AI Guard": zscalerLogo.src, "Presidio PII": microsoftAzureLogo.src, "Bedrock Guardrail": bedrockLogo.src, From daa6e231cbd50b334c67cf0e47b8766ac880ae84 Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Mon, 5 Oct 2026 10:12:49 -0500 Subject: [PATCH 2/9] fix(guardrails): drop dead ThirdLaw suppressions and cover untested branches Removes 18 `# mutable-ok` suppressions that suppress nothing (LIT013), which failed the type-discipline gate. run_in_parallel logged that a modify_response was discarded but then applied it anyway, so a malformed rewrite could still reject a response. It now returns the original response. Adds tests for the fail-open paths, a service failure mid-stream before and after a keepalive flush, sampled-stream refusals, unserializable stream events, and Anthropic citation and malformed-event assembly --- .../guardrail_hooks/thirdlaw/__init__.py | 4 +- .../guardrail_hooks/thirdlaw/thirdlaw.py | 31 ++- .../guardrails/guardrail_hooks/thirdlaw.py | 4 +- .../guardrail_hooks/test_thirdlaw.py | 229 ++++++++++++++++++ .../proxy/guardrails/test_anthropic_sse.py | 59 +++++ 5 files changed, 307 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py index 5ae2f3bee03..0ee92513351 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py @@ -35,10 +35,10 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" return _thirdlaw_callback -guardrail_initializer_registry: Final = { # mutable-ok: the registry loader requires a plain dict +guardrail_initializer_registry: Final = { SupportedGuardrailIntegrations.THIRDLAW.value: initialize_guardrail, } -guardrail_class_registry: Final = { # mutable-ok: the registry loader requires a plain dict +guardrail_class_registry: Final = { SupportedGuardrailIntegrations.THIRDLAW.value: ThirdlawGuardrail, } diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 6761859b509..f124aa5e4c2 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -234,7 +234,7 @@ def _organization_metadata_of(auth: UserAPIKeyAuth) -> object: def _jsonable_dict(value: Mapping[str, object]) -> Mapping[str, object]: """Round-trip through JSON so the payload cannot carry live objects.""" - plain: Final = dict(value) # mutable-ok: json.dumps requires a real dict; consumed immediately + plain: Final = dict(value) return _JSON_DICT_ADAPTER.validate_python(json.loads(json.dumps(plain, default=str))) @@ -540,17 +540,17 @@ class ThirdlawGuardrail(CustomGuardrail): additional_provider_specific_params or _EMPTY_MAP ) - kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) # mutable-ok: base API + kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) super().__init__(**kwargs) self.async_handler = async_handler or get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, - params={"timeout": self.guardrail_timeout}, # mutable-ok: one-shot client-factory argument + params={"timeout": self.guardrail_timeout}, ) @classmethod def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]: # mutable-ok: CustomGuardrail base-class contract - return [ # mutable-ok: CustomGuardrail base-class contract returns a list + return [ GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call, GuardrailEventHooks.during_call, @@ -604,7 +604,7 @@ class ThirdlawGuardrail(CustomGuardrail): ) -> None: now: Final = datetime.now(timezone.utc) self.add_standard_logging_guardrail_information_to_request_data( - guardrail_json_response=dict(trace), # mutable-ok: the logging helper requires a plain dict + guardrail_json_response=dict(trace), request_data=request_data, guardrail_status=status, start_time=started_at.timestamp(), @@ -662,7 +662,7 @@ class ThirdlawGuardrail(CustomGuardrail): try: http_response: Final = await self.async_handler.post( url=self.api_base, - headers=dict(self.http_headers), # mutable-ok: the HTTP client requires a plain dict + headers=dict(self.http_headers), json=payload.model_dump(mode="json", exclude_none=True), ) http_response.raise_for_status() @@ -673,7 +673,7 @@ class ThirdlawGuardrail(CustomGuardrail): event_type=event_type, status="guardrail_failed_to_respond", started_at=started_at, - trace={"error": _service_error_message(error)}, # mutable-ok: one-shot trace payload + trace={"error": _service_error_message(error)}, ) self._handle_call_failure(error=error, wire_event=wire_event) return None @@ -720,17 +720,15 @@ class ThirdlawGuardrail(CustomGuardrail): verbose_proxy_logger.warning( "ThirdLaw guardrail: modify_request decision carried no request_body; request unchanged" ) - return dict(data) # mutable-ok: the pre-call hook contract returns a plain request dict - accepted: Final = { # mutable-ok: one-shot filter merged into the returned request dict - key: value for key, value in replacement.items() if key not in _WRITE_BACK_DENY_KEYS - } + return dict(data) + accepted: Final = {key: value for key, value in replacement.items() if key not in _WRITE_BACK_DENY_KEYS} denied: Final = replacement.keys() - accepted.keys() if denied: verbose_proxy_logger.warning( "ThirdLaw guardrail: dropping protected keys from modify_request write-back: %s", sorted(denied), ) - return {**data, **accepted} # mutable-ok: the proxy owns the replaced request dict + return {**data, **accepted} @staticmethod def _carry_hidden_params(*, source: object, target: object) -> None: @@ -770,7 +768,7 @@ class ThirdlawGuardrail(CustomGuardrail): guardrail_name=self.guardrail_name, message="ThirdLaw guardrail returned a malformed modified response: choices must be a list", ) - merged: Final = { # mutable-ok: one-shot overlay consumed immediately by model_validate + merged: Final = { **_JSON_DICT_ADAPTER.validate_python(response.model_dump()), **replacement, } @@ -785,9 +783,9 @@ class ThirdlawGuardrail(CustomGuardrail): return validated response_dict: Final = _dict_of(response) if response_dict is not None: - return {**response_dict, **replacement} # mutable-ok: the proxy owns the replaced response dict + return {**response_dict, **replacement} if isinstance(response, BaseModel): - merged_body: Final = { # mutable-ok: one-shot overlay consumed immediately by model_validate + merged_body: Final = { **_JSON_DICT_ADAPTER.validate_python(response.model_dump(mode="json")), **replacement, } @@ -910,6 +908,7 @@ class ThirdlawGuardrail(CustomGuardrail): verbose_proxy_logger.warning( "ThirdLaw guardrail: modify_response is discarded when run_in_parallel=True (block-only mode)" ) + return response modified: Final = self._modified_response(response=response, replacement=decision.response_body) return cast(LLMResponseTypes, modified) # cast-ok: dict passthrough mirrors the proxy /v1/messages contract @@ -996,7 +995,7 @@ class ThirdlawGuardrail(CustomGuardrail): from litellm.main import stream_chunk_builder try: - assembled: Final = stream_chunk_builder(chunks=list(collected)) # mutable-ok: builder requires a list + assembled: Final = stream_chunk_builder(chunks=list(collected)) except Exception: # noqa: BLE001 # unassembleable streams route to the explicit fail-closed/pass-through path verbose_proxy_logger.warning("ThirdLaw guardrail: could not assemble streamed response", exc_info=True) return None diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py index 12dfb8b3833..3bfd33c3576 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py @@ -114,8 +114,8 @@ class ThirdlawGuardrailConfigModel(GuardrailConfigModel[ThirdlawGuardrailConfigM api_base: str | None = Field( default=None, description="ThirdLaw intervene-service base URL; /guardrails/litellm/v2 is appended unless already present. Env: THIRDLAW_API_BASE.", - json_schema_extra={ # mutable-ok: pydantic stores schema extras as plain JSON containers - "examples": [ # mutable-ok: pydantic stores schema extras as plain JSON containers + json_schema_extra={ + "examples": [ "https://api.thirdlaw.", ] }, diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index ab325ccafef..4fa7c2b0fce 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -2266,3 +2266,232 @@ async def test_block_never_travels_as_a_success_status(service_status, expected_ call_type="completion", ) assert excinfo.value.status_code == expected_status + + +def _sampled_guardrail(**overrides: object) -> ThirdlawGuardrail: + return _make_guardrail( + streaming_buffer_until_moderated=False, + streaming_end_of_stream_only=False, + **{"streaming_sampling_rate": 100, **overrides}, + ) + + +async def test_request_with_no_inbound_headers_posts_no_headers(): + data = _request_data() + del data["proxy_server_request"]["headers"] + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + await _run_pre_call(g, data) + assert _sent_payload(g).get("request_headers") is None + + +class _SelfReferencingResponse: + """A response whose dump cannot be turned into JSON, the way a live object graph can loop.""" + + def model_dump(self, mode: str = "python") -> JsonDict: + body: JsonDict = {"id": "resp-loop"} + body["self"] = body + return body + + +async def test_post_call_skips_a_response_it_cannot_serialize_instead_of_failing_the_request(): + g = _make_guardrail(decisions=[]) + response = _SelfReferencingResponse() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + assert g.async_handler.post.await_count == 0 + + +class _UndumpableEvent: + type = "response.output_text.delta" + + def model_dump(self, mode: str = "python") -> JsonDict: + raise ValueError("cannot dump") + + +class _ListDumpingEvent: + type = "response.output_text.delta" + + def model_dump(self, mode: str = "python") -> list[object]: + return ["not", "a", "mapping"] + + +async def test_unserializable_stream_events_are_left_out_of_the_posted_chunks(): + """One event that will not serialize must not cost the service the rest of the stream.""" + chunks = _responses_stream_chunks() + stream = [ + *chunks[:2], + {"type": "response.output_text.delta", "delta": " plain-dict"}, + _UndumpableEvent(), + _ListDumpingEvent(), + *chunks[2:], + ] + g = _make_guardrail(send_stream_chunks=True, decisions=[_decision_response({"action": "allow"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(stream), request_data=_request_data() + ) + ) + assert out == stream + posted = _sent_payload(g)["response_chunks"] + assert [c["type"] for c in posted] == [ + "response.created", + "response.output_text.delta", + "response.output_text.delta", + "response.completed", + ] + assert posted[2]["delta"] == " plain-dict" + + +async def test_during_call_passes_through_when_the_service_is_unreachable_and_fail_open(): + g = _make_guardrail(unreachable_fallback="fail_open") + g.async_handler.post.side_effect = _connect_error() + data = _request_data() + out = await g.async_moderation_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), call_type="completion") + assert out is data + assert data["metadata"]["standard_logging_guardrail_information"][0]["guardrail_status"] == ( + "guardrail_failed_to_respond" + ) + + +async def test_post_call_returns_the_original_response_when_the_service_is_unreachable_and_fail_open(): + g = _make_guardrail(unreachable_fallback="fail_open") + g.async_handler.post.side_effect = _connect_error() + response = _model_response() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + + +async def test_run_in_parallel_discards_a_rewrite_without_validating_it(): + """The proxy throws this hook's return value away in parallel mode, so a malformed rewrite + must not be able to reject a response the scan otherwise let through.""" + g = _make_guardrail( + run_in_parallel=True, + decisions=[_decision_response({"action": "modify_response", "response_body": {"choices": "not-a-list"}})], + ) + response = _model_response() + out = await g.async_post_call_success_hook( + data=_request_data(), user_api_key_dict=UserAPIKeyAuth(), response=response + ) + assert out is response + + +async def test_a_buffered_stream_raises_when_the_service_fails_before_headers_are_flushed(): + g = _make_guardrail() + g.async_handler.post.side_effect = _connect_error() + with pytest.raises(GuardrailRaisedException, match="ThirdLaw"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + + +async def test_a_buffered_messages_stream_gets_an_error_frame_when_the_service_fails_after_a_keepalive( + monkeypatch, +): + """Once a keepalive ping has flushed a 200 status line, a raise can no longer reach the + client, so the failure has to travel in-stream as an Anthropic error frame.""" + import litellm + + monkeypatch.setattr(litellm, "anthropic_sse_ping_interval_seconds", 1e-9) + g = _make_guardrail() + g.async_handler.post.side_effect = _connect_error() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_anthropic_sse_frames()), request_data=_request_data() + ) + ) + frames = b"".join(item for item in out if isinstance(item, bytes)).decode() + assert "event: error" in frames + assert "ThirdLaw guardrail request failed" in frames + assert "hello" not in frames + + +async def test_a_buffered_modify_with_no_replacement_body_releases_the_original_stream(): + g = _make_guardrail(decisions=[_decision_response({"action": "modify_response"})]) + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + + +async def test_fail_open_does_not_cover_chat_chunks_that_will_not_assemble(): + """Chat chunks are a supported shape, so failing to assemble them is a scan failure that the + unscannable-stream opt-in must not wave through.""" + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail(decisions=[], unscannable_stream_fallback="fail_open") + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter([*_stream_chunks(), None]), + request_data=_request_data(), + ) + ) + assert g.async_handler.post.await_count == 0 + + +async def test_a_sampled_stream_that_will_not_assemble_is_refused_after_delivery(): + from litellm.proxy.proxy_server import StreamingCallbackError + + first, second, last = _stream_chunks() + g = _sampled_guardrail(streaming_sampling_rate=2, decisions=[]) + agen = g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter([first, None, last]), request_data=_request_data() + ) + assert await agen.__anext__() is first + assert await agen.__anext__() is None + assert await agen.__anext__() is last + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + await agen.__anext__() + assert g.async_handler.post.await_count == 0 + + +async def test_a_sampled_stream_is_delivered_untouched_when_the_service_is_unreachable_and_fail_open(): + g = _sampled_guardrail(unreachable_fallback="fail_open") + g.async_handler.post.side_effect = _connect_error() + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out == chunks + + +async def test_a_sampled_responses_stream_blocked_at_the_end_closes_with_an_error_event(): + """A /v1/responses client only understands Responses events, so the late block has to arrive + as one rather than as the proxy's generic error blob.""" + chunks = _responses_stream_chunks() + g = _sampled_guardrail(decisions=[_decision_response({"action": "block", "message": "final says no"})]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out[: len(chunks)] == chunks + error_event = out[len(chunks)] + assert error_event.type == ResponsesAPIStreamEvents.ERROR + assert "final says no" in str(error_event) + + +async def test_a_sampled_responses_stream_that_never_completes_closes_with_an_error_event(): + chunks = _truncated_responses_chunks() + g = _sampled_guardrail(decisions=[]) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) + ) + assert out[: len(chunks)] == chunks + assert out[len(chunks)].type == ResponsesAPIStreamEvents.ERROR + assert "could not be assembled for scanning" in str(out[len(chunks)]) + assert g.async_handler.post.await_count == 0 diff --git a/tests/unit/proxy/guardrails/test_anthropic_sse.py b/tests/unit/proxy/guardrails/test_anthropic_sse.py index 9e953fbfc13..549bec29fb9 100644 --- a/tests/unit/proxy/guardrails/test_anthropic_sse.py +++ b/tests/unit/proxy/guardrails/test_anthropic_sse.py @@ -104,3 +104,62 @@ def test_a_tool_block_with_no_arguments_assembles_as_an_empty_object(): ("message_stop", {"type": "message_stop"}), ) assert arguments == {} + + +def _text_block_start(index: int = 0) -> tuple[str, dict[str, object]]: + return ( + "content_block_start", + {"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}}, + ) + + +def _block_delta(delta: dict[str, object], index: int = 0) -> tuple[str, dict[str, object]]: + return ("content_block_delta", {"type": "content_block_delta", "index": index, "delta": delta}) + + +def _assembled_content(*events: tuple[str, dict[str, object]]) -> Sequence[dict[str, object]]: + body = assemble_anthropic_sse_body(_frames(*events)) + assert body is not None + content = body["content"] + assert isinstance(content, Sequence) + return content + + +def test_citation_deltas_land_on_the_text_block_they_annotate(): + citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0} + content = _assembled_content( + _message_start(), + _text_block_start(), + _block_delta({"type": "text_delta", "text": "Blue."}), + _block_delta({"type": "citations_delta", "citation": citation}), + _block_delta({"type": "citations_delta"}), + _block_delta({"type": "some_future_delta", "text": "ignored"}), + ) + assert list(content) == [{"type": "text", "text": "Blue.", "citations": (citation,)}] + + +def test_a_block_type_with_no_deltas_keeps_its_start_payload(): + redacted = {"type": "redacted_thinking", "data": "opaque-blob"} + content = _assembled_content( + _message_start(), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": redacted}), + ) + assert list(content) == [redacted] + + +def test_malformed_block_events_are_skipped_without_corrupting_the_rest(): + content = _assembled_content( + _message_start(), + ("content_block_start", {"type": "content_block_start", "content_block": {"type": "text", "text": ""}}), + _text_block_start(), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": "not-a-mapping"}), + _block_delta({"type": "text_delta", "text": "kept"}), + ) + assert list(content) == [{"type": "text", "text": "kept"}] + + +def test_an_undecodable_stream_assembles_to_nothing(): + """Nothing scannable comes out of bytes that are not UTF-8, so callers must treat the stream as + unassembled rather than scanning a partial body.""" + frames = _frames(_message_start(), _text_block_start()) + assert assemble_anthropic_sse_body([*frames[:1], b"\xff\xfe\xfd", *frames[1:]]) is None From 9743c9993c8d987fdd819c725577cd3f7687ed4f Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Mon, 5 Oct 2026 16:45:59 -0500 Subject: [PATCH 3/9] fix(guardrails): keep ThirdLaw streams buffered when masking responses mask_response_content turned streaming_buffer_until_moderated off, so every original chunk reached the client before the scan decided and a modify_response redaction was dropped. The buffered path already emits the rewritten stream, so masking now keeps buffering, the same way Bedrock, Model Armor and PANW handle it Resolves TL-2839 --- .../guardrail_hooks/thirdlaw/thirdlaw.py | 18 +--------- .../guardrail_hooks/test_thirdlaw.py | 34 ++++++++++++++----- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index f124aa5e4c2..44e2c77c495 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -918,7 +918,7 @@ class ThirdlawGuardrail(CustomGuardrail): response: AsyncIterator[object], request_data: dict[str, object], # mutable-ok: proxy-shared request dict; trace records land in it ) -> AsyncGenerator[ModelResponseStream, None]: - buffer: Final = self._buffer_until_moderated() + buffer: Final = self.streaming_buffer_until_moderated end_of_stream_only: Final = buffer or self.streaming_end_of_stream_only # Prefer the stash (see async_post_call_response_headers_hook); `response` here # can be a bare generator with no _hidden_params once another callback is @@ -1001,22 +1001,6 @@ class ThirdlawGuardrail(CustomGuardrail): return None return assembled if isinstance(assembled, ModelResponse) else None - def _buffer_until_moderated(self) -> bool: - """Whether to withhold the stream until the scan decides. - - Buffering replays the original chunks on release, so a guardrail asked to mask content would - hand back the very text it was told to redact. Masking therefore wins over buffering. - """ - requested: Final = self.streaming_buffer_until_moderated - if requested and self.mask_response_content: - verbose_proxy_logger.warning( - "ThirdLaw guardrail: streaming_buffer_until_moderated is disabled for %s because " - "mask_response_content=True -- buffered replay would release unredacted original chunks", - self.guardrail_name, - ) - return False - return requested - def _streaming_block_error(self, message: str) -> Exception: from litellm.proxy.proxy_server import StreamingCallbackError diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 4fa7c2b0fce..974dc10d5b5 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1345,22 +1345,38 @@ def _redacting_modify_decision() -> httpx.Response: ) -async def test_masking_guardrail_does_not_buffer_and_replay_originals(): - """Buffered replay hands back the unredacted chunks, so a masking guardrail must not buffer.""" +def _emitted_text(out: Sequence[object]) -> str: + return "".join( + choice.delta.content or "" + for chunk in out + if isinstance(chunk, ModelResponseStream) + for choice in chunk.choices + ) + + +async def test_masking_guardrail_streams_only_the_redacted_rewrite(): + """Masking must keep the stream buffered: unbuffered, every original chunk reaches the client + before the scan decides and the redaction arrives too late to apply.""" g = _make_guardrail(decisions=[_redacting_modify_decision()], mask_response_content=True) out = await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() ) ) - emitted = "".join( - choice.delta.content or "" - for chunk in out - if isinstance(chunk, ModelResponseStream) - for choice in chunk.choices + emitted = _emitted_text(out) + assert emitted == "the secret is [REDACTED]" + assert "sk-leak" not in emitted + + +async def test_masking_guardrail_releases_the_original_stream_on_allow(): + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})], mask_response_content=True) + chunks = _stream_chunks() + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(chunks), request_data=_request_data() + ) ) - assert emitted == "the secret is sk-leak" - assert "[REDACTED]" not in emitted + assert out == chunks async def test_streaming_buffered_block_raises_streaming_callback_error(): From 22e834f9a9e98f40989b948333624479c287c28d Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Mon, 5 Oct 2026 17:26:13 -0500 Subject: [PATCH 4/9] fix(anthropic): keep citations and redacted_thinking data in replayed Messages streams FakeAnthropicMessagesStreamIterator sent a text block as a single text_delta and ignored its citations, and started a redacted_thinking block without its data. A ThirdLaw rewrite of a /v1/messages stream is replayed through this iterator, so the client lost every citation and could not send a redacted_thinking block back on the next turn Text blocks with citations now start with an empty citations list and send one citations_delta per citation, the way Anthropic's API streams them, and redacted_thinking blocks pass through unchanged Resolves TL-2842 --- .../messages/fake_stream_iterator.py | 29 ++++++++++++++++-- .../guardrail_hooks/test_thirdlaw.py | 30 +++++++++++++++++++ .../proxy/guardrails/test_anthropic_sse.py | 27 ++++++++++++++++- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py index d0fed3225af..b3eb5de8455 100644 --- a/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/pass_through/messages/fake_stream_iterator.py @@ -12,10 +12,14 @@ import json from collections.abc import Mapping from typing import Any, Final, cast +from pydantic import TypeAdapter + from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +_CITATIONS_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) + class FakeAnthropicMessagesStreamIterator: """ @@ -41,14 +45,21 @@ class FakeAnthropicMessagesStreamIterator: def _create_content_block_chunks(self, block_dict: Mapping[str, object], index: int) -> list[bytes]: """Build SSE chunks for a single content block.""" - chunks: Final = [] + chunks: Final[list[bytes]] = [] # mutable-ok: append-built SSE frame buffer, returned as is block_type: Final = block_dict.get("type") if block_type == "text": + raw_citations: Final = block_dict.get("citations") + citations: Final = ( + _CITATIONS_ADAPTER.validate_python(raw_citations) if isinstance(raw_citations, list) else () + ) + text_block: Final[Mapping[str, object]] = ( + {"type": "text", "text": "", "citations": ()} if citations else {"type": "text", "text": ""} + ) content_block_start = { "type": "content_block_start", "index": index, - "content_block": {"type": "text", "text": ""}, + "content_block": text_block, } chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) text: Final = block_dict.get("text", "") @@ -58,6 +69,18 @@ class FakeAnthropicMessagesStreamIterator: "delta": {"type": "text_delta", "text": text}, } chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()) + chunks.extend( + "event: content_block_delta\ndata: {}\n\n".format( + json.dumps( + { + "type": "content_block_delta", + "index": index, + "delta": {"type": "citations_delta", "citation": citation}, + } + ) + ).encode() + for citation in citations + ) elif block_type == "thinking": content_block_start = { @@ -87,7 +110,7 @@ class FakeAnthropicMessagesStreamIterator: content_block_start = { "type": "content_block_start", "index": index, - "content_block": {"type": "redacted_thinking"}, + "content_block": dict(block_dict), } chunks.append(f"event: content_block_start\ndata: {json.dumps(content_block_start)}\n\n".encode()) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 974dc10d5b5..721e82549df 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1673,6 +1673,36 @@ async def test_a_messages_stream_rewrite_is_re_emitted_as_anthropic_frames(): assert b"hello there" not in joined +async def test_a_messages_stream_rewrite_keeps_the_citations_thirdlaw_returned(): + from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_body + + citation = {"type": "char_location", "cited_text": "hello there", "document_index": 0} + g = _make_guardrail( + decisions=[ + _decision_response( + { + "action": "modify_response", + "response_body": { + "content": [{"type": "text", "text": "hello [REDACTED]", "citations": [citation]}] + }, + } + ) + ] + ) + out = await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), + response=_aiter(_anthropic_sse_frames()), + request_data=_request_data(), + ) + ) + replayed = assemble_anthropic_sse_body([item for item in out if isinstance(item, bytes)]) + assert replayed is not None + assert json.loads(json.dumps(replayed["content"])) == [ + {"type": "text", "text": "hello [REDACTED]", "citations": [citation]} + ] + + async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed(): """The overlay is a shallow merge, so "choices" would land beside the untouched "content" and the re-emitted stream would carry the very text the rewrite asked to redact.""" diff --git a/tests/unit/proxy/guardrails/test_anthropic_sse.py b/tests/unit/proxy/guardrails/test_anthropic_sse.py index 549bec29fb9..80227188782 100644 --- a/tests/unit/proxy/guardrails/test_anthropic_sse.py +++ b/tests/unit/proxy/guardrails/test_anthropic_sse.py @@ -8,7 +8,7 @@ of them. import json from collections.abc import Sequence -from litellm.proxy.guardrails.anthropic_sse import assemble_anthropic_sse_body +from litellm.proxy.guardrails.anthropic_sse import anthropic_sse_chunks_from_body, assemble_anthropic_sse_body def _frames(*events: tuple[str, dict[str, object]]) -> list[bytes]: @@ -163,3 +163,28 @@ def test_an_undecodable_stream_assembles_to_nothing(): unassembled rather than scanning a partial body.""" frames = _frames(_message_start(), _text_block_start()) assert assemble_anthropic_sse_body([*frames[:1], b"\xff\xfe\xfd", *frames[1:]]) is None + + +def test_a_replayed_body_keeps_its_citations_and_redacted_thinking(): + """A guardrail rewrite is replayed to the client from the body, so anything the replay drops is + lost to the caller even though the scan saw it: citations lose their sources, and a + redacted_thinking block without its data cannot be sent back on the next turn.""" + citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0} + content = [ + {"type": "redacted_thinking", "data": "opaque-blob"}, + {"type": "text", "text": "The sky is blue.", "citations": [citation]}, + {"type": "text", "text": "No source here."}, + ] + body: dict[str, object] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": content, + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 7}, + } + replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body))) + assert replayed is not None + assert json.loads(json.dumps(replayed["content"])) == content From 763fc0fe8e53ced08dc37a011fe6a398346db8b1 Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Tue, 13 Oct 2026 16:00:49 -0500 Subject: [PATCH 5/9] fix(guardrails): restore the ThirdLaw Messages body assembler lost in the upstream merge The upstream merge resolved anthropic_sse.py to main's version and dropped assemble_anthropic_sse_body, so importing the ThirdLaw guardrail raised ImportError. The file is now main's version plus only the helpers ThirdLaw needs, with main's re-export layout left alone Also clears the lint gate growth main's tighter ceilings exposed in the ThirdLaw code. The streamed body helpers validate nested objects through pydantic adapters instead of reading untyped mappings, the response headers hook returns None to match CustomLogger, get_config_model names its concrete model, the optional params model is frozen, and untyped upstream helpers carry reasoned ignores --- litellm/proxy/guardrails/anthropic_sse.py | 71 ++++++++++++++----- .../guardrail_hooks/thirdlaw/__init__.py | 2 +- .../guardrail_hooks/thirdlaw/thirdlaw.py | 44 ++++++------ litellm/proxy/guardrails/stream_surface.py | 8 ++- .../guardrails/guardrail_hooks/thirdlaw.py | 2 + 5 files changed, 84 insertions(+), 43 deletions(-) diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 7e9df16501d..5ebd814f979 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -18,6 +18,8 @@ from typing import ( cast, # noqa: TID251 # the Messages body is a total=False TypedDict; validating through it drops provider fields ) +from pydantic import TypeAdapter + from litellm.llms.anthropic.pass_through.stream_assembly import ( _joined_sse_stream, # pyright: ignore[reportPrivateUsage] # moved with the assembler; a copy splits trailing utf-8 _parsed_sse_events, # pyright: ignore[reportPrivateUsage] # same parser assemble_anthropic_sse_stream uses @@ -31,9 +33,6 @@ from litellm.llms.anthropic.pass_through.stream_assembly import ( from litellm.llms.anthropic.pass_through.stream_assembly import ( is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export ) -from litellm.llms.anthropic.pass_through.stream_assembly import ( - is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export -) from litellm.types.utils import Choices, ModelResponse if TYPE_CHECKING: @@ -43,6 +42,8 @@ _EMPTY_MAP: Final[Mapping[str, object]] = MappingProxyType({}) _TOOL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"}) +_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object]) + def sse_stream_text(all_chunks: Sequence[object]) -> str | None: """The raw SSE frames joined as text, for a caller that forwards the stream instead of folding it.""" @@ -72,6 +73,11 @@ def anthropic_sse_error_frames(message: str) -> tuple[bytes, ...]: ) +from litellm.llms.anthropic.pass_through.stream_assembly import ( # noqa: E402 # re-export after the definitions above + is_sse_error_stream as is_sse_error_stream, # noqa: PLC0414 # public re-export +) + + def anthropic_sse_chunks_from_response(assembled: ModelResponse) -> tuple[bytes, ...]: from litellm.llms.anthropic.pass_through.adapters.transformation import ( LiteLLMAnthropicMessagesAdapter, @@ -90,6 +96,11 @@ def _str_field(source: Mapping[str, object], key: str) -> str: return value if isinstance(value := source.get(key), str) else "" +def _object_field(source: Mapping[str, object], key: str) -> Mapping[str, object] | None: + value: Final = source.get(key) + return _JSON_OBJECT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else None + + def _accumulated_tool_input(partial_json: Sequence[str]) -> object: """Assemble a tool block's streamed arguments. @@ -146,7 +157,7 @@ class _ContentBlock: return {"input": _accumulated_tool_input(self.partial_json)} return _EMPTY_MAP - def rendered(self) -> dict[str, object]: + def rendered(self) -> Mapping[str, object]: cited: Final[Mapping[str, object]] = ( MappingProxyType({"citations": tuple(self.citations)}) if self.citations else _EMPTY_MAP ) @@ -170,34 +181,31 @@ def _with_block(state: _AnthropicMessage, index: int, block: _ContentBlock) -> _ def _with_message_delta(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage: - delta: Final = event.get("delta") - usage: Final = event.get("usage") - stops: Final = delta if isinstance(delta, Mapping) else _EMPTY_MAP + stops: Final = _object_field(event, "delta") or _EMPTY_MAP + usage: Final = _object_field(event, "usage") return replace( state, stop_reason=stops.get("stop_reason", state.stop_reason), stop_sequence=stops.get("stop_sequence", state.stop_sequence), - delta_usage=( - MappingProxyType({**state.delta_usage, **usage}) if isinstance(usage, Mapping) else state.delta_usage - ), + delta_usage=MappingProxyType({**state.delta_usage, **usage}) if usage is not None else state.delta_usage, ) def _with_event(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage: match event.get("type"): case "message_start": - message: Final = event.get("message") - return replace(state, envelope=message) if isinstance(message, Mapping) else state + message: Final = _object_field(event, "message") + return state if message is None else replace(state, envelope=message) case "content_block_start": index: Final = event.get("index") - block: Final = event.get("content_block") - if not isinstance(index, int) or not isinstance(block, Mapping): + block: Final = _object_field(event, "content_block") + if not isinstance(index, int) or block is None: return state return _with_block(state, index, _ContentBlock(start=block)) case "content_block_delta": delta_index: Final = event.get("index") - delta: Final = event.get("delta") - if not isinstance(delta_index, int) or not isinstance(delta, Mapping): + delta: Final = _object_field(event, "delta") + if not isinstance(delta_index, int) or delta is None: return state started: Final = state.blocks.get(delta_index) return state if started is None else _with_block(state, delta_index, started.with_delta(delta)) @@ -206,11 +214,40 @@ def _with_event(state: _AnthropicMessage, event: Mapping[str, object]) -> _Anthr case _: return state + +def assemble_anthropic_sse_body(all_chunks: Sequence[object]) -> Mapping[str, object] | None: + """Fold raw Anthropic SSE frames back into the non-streaming Messages body. + + ``assemble_anthropic_sse_stream`` folds them into a ``ModelResponse``, which is all a guardrail + scanning text needs. A guardrail that posts the provider body itself needs the shape the + non-streaming route already sends, because the chat-completions one has no field for thinking + blocks, citations, ``stop_sequence`` or the cache token split and drops them silently. + """ + sse_stream: Final = _joined_sse_stream(all_chunks) + if sse_stream is None: + return None + state: Final = reduce(_with_event, _parsed_sse_events(sse_stream), _AnthropicMessage()) + envelope: Final = state.envelope + if envelope is None: + return None + started_usage: Final = envelope.get("usage") + return { + **envelope, + "content": tuple(block.rendered() for _, block in sorted(state.blocks.items())), + "stop_reason": state.stop_reason if state.stop_reason is not None else envelope.get("stop_reason"), + "stop_sequence": state.stop_sequence if state.stop_sequence is not None else envelope.get("stop_sequence"), + "usage": { + **(started_usage if isinstance(started_usage, Mapping) else _EMPTY_MAP), + **state.delta_usage, + }, + } + + def anthropic_sse_chunks_from_body(body: Mapping[str, object]) -> tuple[bytes, ...]: """Re-emit a native Anthropic Messages body as the SSE frames its client expects.""" from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) - typed: Final = cast("AnthropicMessagesResponse", dict(body)) + typed: Final = cast("AnthropicMessagesResponse", dict(body)) # cast-ok: validating drops provider-only fields return tuple(FakeAnthropicMessagesStreamIterator(response=typed).chunks) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py index 0ee92513351..3006cae714a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/__init__.py @@ -27,7 +27,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", None), headers=getattr(litellm_params, "headers", None), extra_headers=getattr(litellm_params, "extra_headers", None), - guardrail_name=guardrail.get("guardrail_name", ""), + guardrail_name=guardrail["guardrail_name"], event_hook=litellm_params.mode, default_on=litellm_params.default_on, ) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 44e2c77c495..fbd9c521541 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -4,7 +4,6 @@ from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence from datetime import datetime, timezone from types import MappingProxyType from typing import ( - TYPE_CHECKING, Final, Literal, TypeAlias, @@ -22,11 +21,11 @@ from litellm.exceptions import GuardrailRaisedException from litellm.exceptions import Timeout as LitellmTimeout from litellm.integrations.custom_guardrail import ( CustomGuardrail, - log_guardrail_information, + log_guardrail_information, # pyright: ignore[reportUnknownVariableType] # decorator is untyped in custom_guardrail ) from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, - get_async_httpx_client, + get_async_httpx_client, # pyright: ignore[reportUnknownVariableType] # helper is untyped in http_handler httpxSpecialProvider, ) from litellm.proxy._types import ( @@ -55,6 +54,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.guardrails import GuardrailEventHooks from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( + ThirdlawGuardrailConfigModel, ThirdlawGuardrailRequest, ThirdlawGuardrailRequestMetadata, ThirdlawGuardrailResponse, @@ -67,9 +67,6 @@ from litellm.types.utils import ( ModelResponseStream, ) -if TYPE_CHECKING: - from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel - GUARDRAIL_NAME: Final = "thirdlaw" _ENDPOINT_PATH: Final = "/guardrails/litellm/v2" @@ -192,6 +189,8 @@ _AssembledStream: TypeAlias = ModelResponse | ResponsesAPIResponse | Mapping[str _RESPONSE_WRITE_BACK_DENY_KEYS: Final = frozenset({"id"}) _JSON_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) +_OBJECT_MAP_ADAPTER: Final = TypeAdapter(dict[object, object]) +_STR_MAP_ADAPTER: Final = TypeAdapter(dict[str, str]) # tags metadata rides in as an untyped list; validating through a TypeAdapter (its # `validate_python` parameter is `Any`, so it accepts an untyped value without complaint) @@ -359,13 +358,13 @@ def _response_headers(response: object) -> Mapping[str, str] | None: """ hidden_params: Final[object] = getattr(response, "_hidden_params", None) headers: Final[object] = ( - hidden_params.get("additional_headers") + _OBJECT_MAP_ADAPTER.validate_python(hidden_params).get("additional_headers") if isinstance(hidden_params, Mapping) else getattr(hidden_params, "additional_headers", None) ) if not isinstance(headers, Mapping) or not headers: return None - return MappingProxyType({str(k): str(v) for k, v in headers.items()}) + return MappingProxyType({str(k): str(v) for k, v in _OBJECT_MAP_ADAPTER.validate_python(headers).items()}) # Where async_post_call_response_headers_hook stashes response headers for the @@ -376,7 +375,7 @@ _RESPONSE_HEADERS_STASH_KEY: Final = "_thirdlaw_response_headers" def _stashed_response_headers(request_data: Mapping[str, object]) -> Mapping[str, str] | None: stashed: Final = request_data.get(_RESPONSE_HEADERS_STASH_KEY) - return stashed if isinstance(stashed, Mapping) else None + return _STR_MAP_ADAPTER.validate_python(stashed) if isinstance(stashed, Mapping) else None def _request_body(request_data: Mapping[str, object], prefer_snapshot: bool) -> Mapping[str, object] | None: @@ -502,7 +501,7 @@ class ThirdlawGuardrail(CustomGuardrail): headers: Mapping[str, str] | None = None, extra_headers: Sequence[str] | None = None, async_handler: AsyncHTTPHandler | None = None, - **kwargs, # noqa: ANN003 # kwargs-ok: forwarded verbatim to CustomGuardrail, which owns their types + **kwargs: object, # kwargs-ok: forwarded verbatim to CustomGuardrail, which owns their types ) -> None: resolved_base: Final = api_base or get_secret_str("THIRDLAW_API_BASE") if not resolved_base: @@ -541,7 +540,7 @@ class ThirdlawGuardrail(CustomGuardrail): ) kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) - super().__init__(**kwargs) + super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # untyped base self.async_handler = async_handler or get_async_httpx_client( llm_provider=httpxSpecialProvider.GuardrailCallback, @@ -557,11 +556,7 @@ class ThirdlawGuardrail(CustomGuardrail): ] @staticmethod - def get_config_model() -> type["GuardrailConfigModel"] | None: - from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import ( - ThirdlawGuardrailConfigModel, - ) - + def get_config_model() -> type[ThirdlawGuardrailConfigModel] | None: return ThirdlawGuardrailConfigModel def _build_wire_request( @@ -736,9 +731,9 @@ class ThirdlawGuardrail(CustomGuardrail): attributes -- ``_hidden_params`` (which carries ``additional_headers`` for response- header forwarding) is one of those, so copy it across explicitly. """ - hidden_params = getattr(source, "_hidden_params", None) + hidden_params: Final = getattr(source, "_hidden_params", None) if hidden_params is not None: - setattr(target, "_hidden_params", hidden_params) # target's concrete type varies by call site + setattr(target, "_hidden_params", hidden_params) # noqa: B010 # target is typed object; the type varies by call site def _without_response_id(self, replacement: Mapping[str, object]) -> Mapping[str, object]: """Drop ``id`` from a modify_response write-back. @@ -806,7 +801,7 @@ class ThirdlawGuardrail(CustomGuardrail): def _mark_applied(self, request_data: dict[str, object]) -> None: # mutable-ok: header helper appends to metadata from litellm.proxy.common_utils.callback_utils import ( - add_guardrail_to_applied_guardrails_header, + add_guardrail_to_applied_guardrails_header, # pyright: ignore[reportUnknownVariableType] # helper is untyped ) add_guardrail_to_applied_guardrails_header(request_data=request_data, guardrail_name=self.guardrail_name) @@ -818,18 +813,17 @@ class ThirdlawGuardrail(CustomGuardrail): response: object, request_headers: Mapping[str, str] | None = None, litellm_call_info: Mapping[str, object] | None = None, - ) -> Mapping[str, str] | None: + ) -> None: """Capture the provider's response headers off the true, unwrapped response. A guardrail chained ahead of ThirdLaw can obscure ``_hidden_params`` on the ``response`` the streaming hook later receives (``ResponsesIDSecurity``, e.g., hands back a bare generator with none); this hook always gets the real thing. - Always returns None -- it only stashes for ThirdLaw's own later use. + It only stashes them for ThirdLaw's own later use. """ headers: Final = _response_headers(response) if headers is not None: - data[_RESPONSE_HEADERS_STASH_KEY] = headers - return None + data[_RESPONSE_HEADERS_STASH_KEY] = headers # rebind-ok: only channel to the stream hook @log_guardrail_information async def async_pre_call_hook( @@ -992,7 +986,9 @@ class ThirdlawGuardrail(CustomGuardrail): @staticmethod def _assembled_chat_stream_response(collected: Sequence[object]) -> ModelResponse | None: - from litellm.main import stream_chunk_builder + from litellm.main import ( + stream_chunk_builder, # pyright: ignore[reportUnknownVariableType] # builder is partially untyped + ) try: assembled: Final = stream_chunk_builder(chunks=list(collected)) diff --git a/litellm/proxy/guardrails/stream_surface.py b/litellm/proxy/guardrails/stream_surface.py index 3167940bda1..95d9dfb6eb2 100644 --- a/litellm/proxy/guardrails/stream_surface.py +++ b/litellm/proxy/guardrails/stream_surface.py @@ -15,6 +15,8 @@ from collections.abc import Sequence from enum import Enum, auto from typing import Final +from pydantic import TypeAdapter + from litellm.proxy.guardrails.anthropic_sse import ( is_anthropic_sse_stream, is_raw_sse_stream, @@ -24,6 +26,8 @@ from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamEv _RESPONSES_EVENT_TYPE_PREFIX: Final = "response." +_OBJECT_MAP_ADAPTER: Final = TypeAdapter(dict[object, object]) + # Only these carry the finished output; response.created also carries a body, but an empty one _RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset( { @@ -54,7 +58,9 @@ class StreamSurface(Enum): def _stream_item_field(item: object, field: str) -> object | None: - return item.get(field) if isinstance(item, dict) else getattr(item, field, None) + return ( + _OBJECT_MAP_ADAPTER.validate_python(item).get(field) if isinstance(item, dict) else getattr(item, field, None) + ) def _stream_item_type(item: object) -> str | None: diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py index 3bfd33c3576..60ea440514b 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py @@ -76,6 +76,8 @@ class ThirdlawGuardrailResponse(BaseModel): class ThirdlawGuardrailConfigModelOptionalParams(BaseModel): + model_config = ConfigDict(frozen=True) + additional_headers: str | None = Field( default=None, description="Comma-separated list of inbound request header names whose RAW values ThirdLaw should receive. Every inbound header is always forwarded; credential headers (authorization, x-api-key, cookie) are forwarded as ***REDACTED*** unless listed here. Example: authorization,x-request-id.", From b17b9249be4a1a76055d7773a453f092c1dae943 Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Tue, 13 Oct 2026 16:37:02 -0500 Subject: [PATCH 6/9] fix(guardrails): close three ThirdLaw scan and credential gaps from bot review A tool block that arrives whole in content_block_start, as server_tool_use does, kept no input after assembly, so its query was replayed to the client without being scanned. The assembler now keeps the start's input when no argument deltas follow The response headers stash was only stripped of caller-planted values by the pre-call hook, which the proxy skips for a post_call-only guardrail. The stash is now a private dataclass that a JSON request body cannot produce, so a planted value is ignored whichever hooks run request_url carried the Google routes' ?key= credential to ThirdLaw unchanged. Credential query parameters are now redacted with the same placeholder the proxy uses for headers --- litellm/proxy/guardrails/anthropic_sse.py | 8 ++- .../guardrail_hooks/thirdlaw/thirdlaw.py | 32 ++++++++++-- .../guardrail_hooks/test_thirdlaw.py | 50 ++++++++++++++++--- .../proxy/guardrails/test_anthropic_sse.py | 21 ++++++++ 4 files changed, 95 insertions(+), 16 deletions(-) diff --git a/litellm/proxy/guardrails/anthropic_sse.py b/litellm/proxy/guardrails/anthropic_sse.py index 5ebd814f979..562b5310e61 100644 --- a/litellm/proxy/guardrails/anthropic_sse.py +++ b/litellm/proxy/guardrails/anthropic_sse.py @@ -101,7 +101,7 @@ def _object_field(source: Mapping[str, object], key: str) -> Mapping[str, object return _JSON_OBJECT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else None -def _accumulated_tool_input(partial_json: Sequence[str]) -> object: +def _accumulated_tool_input(joined: str) -> object: """Assemble a tool block's streamed arguments. A stream cut mid-block leaves unparseable JSON. Returning an empty input there would @@ -111,9 +111,6 @@ def _accumulated_tool_input(partial_json: Sequence[str]) -> object: So the raw fragment is carried under ``_raw`` instead, keeping the block a well-formed object while leaving the text visible to whatever inspects the assembled body. """ - joined: Final = "".join(partial_json) - if not joined: - return {} try: return json.loads(joined) except json.JSONDecodeError: @@ -154,7 +151,8 @@ class _ContentBlock: if block_type == "thinking": return {"thinking": "".join(self.thinking), "signature": "".join(self.signature)} if block_type in _TOOL_BLOCK_TYPES: - return {"input": _accumulated_tool_input(self.partial_json)} + joined: Final = "".join(self.partial_json) + return {"input": _accumulated_tool_input(joined)} if joined else _EMPTY_MAP return _EMPTY_MAP def rendered(self) -> Mapping[str, object]: diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index fbd9c521541..61cefcd03da 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -1,6 +1,7 @@ import json import time from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence +from dataclasses import dataclass from datetime import datetime, timezone from types import MappingProxyType from typing import ( @@ -9,6 +10,7 @@ from typing import ( TypeAlias, cast, # noqa: TID251 # SSE byte frames ride the ModelResponseStream-typed pipe (bedrock precedent) ) +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import httpx from pydantic import BaseModel, TypeAdapter, ValidationError @@ -47,6 +49,7 @@ from litellm.proxy.guardrails.stream_surface import ( responses_deltas_absent_from_body, ) from litellm.proxy.litellm_pre_call_utils import ( + _REDACTED_HEADER_VALUE, # pyright: ignore[reportPrivateUsage] # same placeholder the proxy redacts headers to _UNTRUSTED_METADATA_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # shared list _UNTRUSTED_ROOT_CONTROL_FIELDS, # pyright: ignore[reportPrivateUsage] # shared list ) @@ -190,7 +193,6 @@ _RESPONSE_WRITE_BACK_DENY_KEYS: Final = frozenset({"id"}) _JSON_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _OBJECT_MAP_ADAPTER: Final = TypeAdapter(dict[object, object]) -_STR_MAP_ADAPTER: Final = TypeAdapter(dict[str, str]) # tags metadata rides in as an untyped list; validating through a TypeAdapter (its # `validate_python` parameter is `Any`, so it accepts an untyped value without complaint) @@ -291,9 +293,23 @@ def _proxy_server_request(request_data: Mapping[str, object]) -> Mapping[str, ob return _dict_of(request_data.get("proxy_server_request")) or _EMPTY_MAP +_CREDENTIAL_QUERY_PARAMS: Final = frozenset({"key"}) + + +def _without_query_credentials(url: str) -> str: + parts: Final = urlsplit(url) + params: Final = parse_qsl(parts.query, keep_blank_values=True) + if not any(name.lower() in _CREDENTIAL_QUERY_PARAMS for name, _ in params): + return url + redacted: Final = tuple( + (name, _REDACTED_HEADER_VALUE if name.lower() in _CREDENTIAL_QUERY_PARAMS else value) for name, value in params + ) + return urlunsplit(parts._replace(query=urlencode(redacted, safe="*"))) + + def _request_url(request_data: Mapping[str, object]) -> str | None: url: Final = _proxy_server_request(request_data).get("url") - return url if isinstance(url, str) else None + return _without_query_credentials(url) if isinstance(url, str) else None def _redacted_inbound_headers(request_data: Mapping[str, object]) -> Mapping[str, object] | None: @@ -373,9 +389,16 @@ def _response_headers(response: object) -> Mapping[str, str] | None: _RESPONSE_HEADERS_STASH_KEY: Final = "_thirdlaw_response_headers" +@dataclass(frozen=True, slots=True) +class _StashedResponseHeaders: + """A caller's JSON body only decodes to plain types, so it can never plant one of these.""" + + headers: Mapping[str, str] + + def _stashed_response_headers(request_data: Mapping[str, object]) -> Mapping[str, str] | None: stashed: Final = request_data.get(_RESPONSE_HEADERS_STASH_KEY) - return _STR_MAP_ADAPTER.validate_python(stashed) if isinstance(stashed, Mapping) else None + return stashed.headers if isinstance(stashed, _StashedResponseHeaders) else None def _request_body(request_data: Mapping[str, object], prefer_snapshot: bool) -> Mapping[str, object] | None: @@ -823,7 +846,8 @@ class ThirdlawGuardrail(CustomGuardrail): """ headers: Final = _response_headers(response) if headers is not None: - data[_RESPONSE_HEADERS_STASH_KEY] = headers # rebind-ok: only channel to the stream hook + stashed: Final = _StashedResponseHeaders(headers) + data[_RESPONSE_HEADERS_STASH_KEY] = stashed # rebind-ok: only channel to the stream hook @log_guardrail_information async def async_pre_call_hook( diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 721e82549df..e0af711384b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -806,7 +806,7 @@ async def test_response_headers_hook_stashes_headers_and_returns_none(): """async_post_call_response_headers_hook must never inject anything into the client-facing response -- it exists purely to capture headers off the true response object for ThirdLaw's own later use.""" - g = _make_guardrail() + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) response = _model_response() response._hidden_params["additional_headers"] = {"x-request-id": "req-1"} data = _request_data() @@ -814,7 +814,8 @@ async def test_response_headers_hook_stashes_headers_and_returns_none(): data=data, user_api_key_dict=UserAPIKeyAuth(), response=response ) assert out is None - assert data["_thirdlaw_response_headers"] == {"x-request-id": "req-1"} + await g.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=_model_response()) + assert _sent_payload(g)["response_headers"] == {"x-request-id": "req-1"} async def test_response_headers_hook_leaves_stash_untouched_when_absent(): @@ -830,10 +831,12 @@ async def test_post_call_success_hook_prefers_the_stash_over_the_response_object """When both are present, the response-headers-hook stash wins -- it is the reliable source; a direct read off `response` is the fallback.""" g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + unwrapped = _model_response() + unwrapped._hidden_params["additional_headers"] = {"x-request-id": "from-stash"} response = _model_response() response._hidden_params["additional_headers"] = {"x-request-id": "from-response-object"} data = _request_data() - data["_thirdlaw_response_headers"] = {"x-request-id": "from-stash"} + await g.async_post_call_response_headers_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=unwrapped) await g.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=response) assert _sent_payload(g)["response_headers"] == {"x-request-id": "from-stash"} @@ -852,11 +855,11 @@ async def test_pre_call_strips_a_caller_forged_response_headers_stash(): async def test_streaming_never_trusts_a_caller_forged_response_headers_stash(): """A caller-forged stash entry must not survive to the wire payload, even when the stream itself carries no real headers (e.g. another callback's bare generator, as in - test_streaming_falls_back_to_the_stash_when_response_carries_no_hidden_params).""" - g = _make_guardrail(decisions=[_decision_response({"action": "allow"}), _decision_response({"action": "allow"})]) + test_streaming_falls_back_to_the_stash_when_response_carries_no_hidden_params). A + post_call-only guardrail never runs its pre-call hook, so nothing strips the field first.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) data = _request_data() data["_thirdlaw_response_headers"] = {"x-forged-by": "the caller"} - await _run_pre_call(g, data) # the real proxy always runs this first async def bare_generator_with_no_hidden_params(): for chunk in _stream_chunks(): @@ -872,6 +875,35 @@ async def test_streaming_never_trusts_a_caller_forged_response_headers_stash(): assert "response_headers" not in _sent_payload(g) +async def test_post_call_only_never_trusts_a_caller_forged_response_headers_stash(): + """Without a pre-call run, the headers hook finding no provider headers must not leave the + caller's planted value to be posted as if the provider had sent it.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + data["_thirdlaw_response_headers"] = {"x-forged-by": "the caller"} + await g.async_post_call_response_headers_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(), response=_model_response() + ) + await g.async_post_call_success_hook(data=data, user_api_key_dict=UserAPIKeyAuth(), response=_model_response()) + assert "response_headers" not in _sent_payload(g) + + +async def test_query_string_credentials_are_redacted_from_the_request_url(): + """Google routes authenticate with ``?key=``, so the URL itself carries the LiteLLM key.""" + g = _make_guardrail(decisions=[_decision_response({"action": "allow"})]) + data = _request_data() + data["proxy_server_request"] = { + **data["proxy_server_request"], + "url": "http://localhost:4000/v1beta/models/gemini-pro:streamGenerateContent?alt=sse&KEY=sk-caller-key", + } + await _run_pre_call(g, data) + payload = _sent_payload(g) + assert "sk-caller-key" not in json.dumps(payload) + assert payload["request_url"] == ( + "http://localhost:4000/v1beta/models/gemini-pro:streamGenerateContent?alt=sse&KEY=***REDACTED***" + ) + + async def test_post_call_block_raises(): g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) with pytest.raises(GuardrailRaisedException) as exc_info: @@ -1264,8 +1296,12 @@ async def test_streaming_falls_back_to_the_stash_when_response_carries_no_hidden for chunk in _stream_chunks(): yield chunk + unwrapped = _model_response() + unwrapped._hidden_params["additional_headers"] = {"x-request-id": "req-78"} request_data = _request_data() - request_data["_thirdlaw_response_headers"] = {"x-request-id": "req-78"} + await g.async_post_call_response_headers_hook( + data=request_data, user_api_key_dict=UserAPIKeyAuth(), response=unwrapped + ) await _collect( g.async_post_call_streaming_iterator_hook( diff --git a/tests/unit/proxy/guardrails/test_anthropic_sse.py b/tests/unit/proxy/guardrails/test_anthropic_sse.py index 80227188782..ce3f4b45388 100644 --- a/tests/unit/proxy/guardrails/test_anthropic_sse.py +++ b/tests/unit/proxy/guardrails/test_anthropic_sse.py @@ -188,3 +188,24 @@ def test_a_replayed_body_keeps_its_citations_and_redacted_thinking(): replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body))) assert replayed is not None assert json.loads(json.dumps(replayed["content"])) == content + + +def test_a_tool_input_sent_whole_in_its_block_start_survives_assembly(): + """Server tool blocks reach the stream with their full input in ``content_block_start`` and no + argument deltas. Blanking that input would replay a query to the client that was never scanned.""" + body: dict[str, object] = { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "restricted"}} + ], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 7}, + } + replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body))) + assert replayed is not None + assert json.loads(json.dumps(replayed["content"])) == body["content"] + From 83a061d17506be92a1d08bbc4c490c1ed206c8f5 Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Thu, 29 Oct 2026 22:47:18 -0500 Subject: [PATCH 7/9] fix(guardrails): answer a ThirdLaw streamed chat block with its block status A block decided at the end of a buffered Chat Completions stream raised a bare StreamingCallbackError. The proxy serializes an exception without a status as code 500, and because nothing had reached the client yet, create_response sent an HTTP 500 instead of the 403 ThirdLaw rendered StreamingCallbackError now takes an optional status_code, and every streamed block passes the decision's response_status through _block_status, the same mapping the unary path uses. Other guardrails raise it without a status and still answer 500 Part of TL-2934 --- .../guardrail_hooks/thirdlaw/thirdlaw.py | 13 ++++--- litellm/proxy/proxy_server.py | 7 +++- .../guardrail_hooks/test_thirdlaw.py | 37 ++++++++++++++++--- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 61cefcd03da..1048d0177c1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -1021,10 +1021,10 @@ class ThirdlawGuardrail(CustomGuardrail): return None return assembled if isinstance(assembled, ModelResponse) else None - def _streaming_block_error(self, message: str) -> Exception: + def _streaming_block_error(self, message: str, *, response_status: int | None = None) -> Exception: from litellm.proxy.proxy_server import StreamingCallbackError - return StreamingCallbackError(message) + return StreamingCallbackError(message, status_code=self._block_status(response_status)) @staticmethod def _stream_error_items( @@ -1138,7 +1138,7 @@ class ThirdlawGuardrail(CustomGuardrail): message: Final = decision.message or "Content violates ThirdLaw policy" block_items: Final = self._stream_error_items(message, surface, collected) if block_items is None: - raise self._streaming_block_error(message) + raise self._streaming_block_error(message, response_status=decision.response_status) for item in block_items: yield item return @@ -1289,7 +1289,10 @@ class ThirdlawGuardrail(CustomGuardrail): response_headers=response_headers, ) if interim_decision is not None and interim_decision.action == "block": - raise self._streaming_block_error(interim_decision.message or "Content violates ThirdLaw policy") + raise self._streaming_block_error( + interim_decision.message or "Content violates ThirdLaw policy", + response_status=interim_decision.response_status, + ) surface: Final = classify_stream(collected) if surface is StreamSurface.RESPONSES: @@ -1322,7 +1325,7 @@ class ThirdlawGuardrail(CustomGuardrail): message: Final = final_decision.message or "Content violates ThirdLaw policy" block_items: Final = self._stream_error_items(message, surface, collected) if block_items is None: - raise self._streaming_block_error(message) + raise self._streaming_block_error(message, response_status=final_decision.response_status) for item in block_items: yield item return diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 78187a59e5b..eccc7ae539f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4854,7 +4854,12 @@ async def _run_background_health_check(): class StreamingCallbackError(Exception): - pass + status_code: int + + def __init__(self, message: str, *, status_code: int | None = None) -> None: + super().__init__(message) + if status_code is not None: + self.status_code = status_code # Fields in ``litellm_settings`` / ``general_settings`` whose values flow diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index e0af711384b..6b3d4b7d12e 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1419,12 +1419,34 @@ async def test_streaming_buffered_block_raises_streaming_callback_error(): from litellm.proxy.proxy_server import StreamingCallbackError g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) - with pytest.raises(StreamingCallbackError, match="leaked secret"): + with pytest.raises(StreamingCallbackError, match="leaked secret") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() ) ) + assert raised.value.status_code == 400 + + +@pytest.mark.parametrize( + ("response_status", "expected_status"), + [(200, 400), (422, 422), (503, 503)], +) +async def test_streaming_buffered_block_raises_with_the_block_status(response_status: int, expected_status: int): + from litellm.proxy.proxy_server import StreamingCallbackError + + g = _make_guardrail( + decisions=[ + _decision_response({"action": "block", "message": "leaked secret", "response_status": response_status}) + ] + ) + with pytest.raises(StreamingCallbackError, match="leaked secret") as raised: + await _collect( + g.async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() + ) + ) + assert raised.value.status_code == expected_status async def test_streaming_buffered_holds_chunks_until_decision(): @@ -1458,15 +1480,16 @@ async def test_streaming_sampled_interim_block_terminates_stream(): streaming_buffer_until_moderated=False, streaming_end_of_stream_only=False, streaming_sampling_rate=1, - decisions=[_decision_response({"action": "block", "message": "bad interim"})], + decisions=[_decision_response({"action": "block", "message": "bad interim", "response_status": 422})], ) agen = g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_stream_chunks()), request_data=_request_data() ) first = await agen.__anext__() assert isinstance(first, ModelResponseStream) - with pytest.raises(StreamingCallbackError, match="bad interim"): + with pytest.raises(StreamingCallbackError, match="bad interim") as raised: await agen.__anext__() + assert raised.value.status_code == 422 def _truncated_tool_use_sse_frames() -> list[bytes]: @@ -1798,7 +1821,7 @@ async def test_unscannable_stream_fails_closed_by_default(): from litellm.proxy.proxy_server import StreamingCallbackError g = _make_guardrail(decisions=[]) - with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning"): + with pytest.raises(StreamingCallbackError, match="could not be assembled for scanning") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1806,6 +1829,7 @@ async def test_unscannable_stream_fails_closed_by_default(): request_data=_request_data(), ) ) + assert raised.value.status_code == 400 assert g.async_handler.post.await_count == 0 @@ -1905,9 +1929,9 @@ async def test_a_sampled_stream_blocks_on_the_final_decision(): streaming_buffer_until_moderated=False, streaming_end_of_stream_only=False, streaming_sampling_rate=100, - decisions=[_decision_response({"action": "block", "message": "final says no"})], + decisions=[_decision_response({"action": "block", "message": "final says no", "response_status": 451})], ) - with pytest.raises(StreamingCallbackError, match="final says no"): + with pytest.raises(StreamingCallbackError, match="final says no") as raised: await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1915,6 +1939,7 @@ async def test_a_sampled_stream_blocks_on_the_final_decision(): request_data=_request_data(), ) ) + assert raised.value.status_code == 451 async def test_a_sampled_stream_cannot_apply_a_late_modify_response(): From be649272d47ae5c9e05e454a0108fd2821002a3c Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Thu, 29 Oct 2026 22:52:14 -0500 Subject: [PATCH 8/9] test(guardrails): pin that a ThirdLaw streamed block reaches the client as its status The streamed block tests only read status_code off the raised exception, so they still passed if the proxy dropped it on the way out, which is how LibreChat got an HTTP 500 for a 403 block. This one runs the real guardrail through async_data_generator and create_response and checks the HTTP status and message the client receives. It fails with 500 if either the guardrail or the proxy loses the status Part of TL-2934 --- .../guardrail_hooks/test_thirdlaw.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 6b3d4b7d12e..4819b139f70 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1449,6 +1449,33 @@ async def test_streaming_buffered_block_raises_with_the_block_status(response_st assert raised.value.status_code == expected_status +async def test_a_buffered_streamed_block_reaches_the_client_as_its_block_status(monkeypatch: pytest.MonkeyPatch): + """The status has to survive the proxy's stream error path, not only sit on the exception. + Dropping it answered LibreChat with an HTTP 500 for a 403 block (TL-2934).""" + import litellm + from litellm.proxy.common_request_processing import create_response + from litellm.proxy.proxy_server import ( + async_data_generator, # pyright: ignore[reportUnknownVariableType] # its response parameter is untyped + ) + + g = _make_guardrail( + event_hook="post_call", + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})], + ) + monkeypatch.setattr(litellm, "callbacks", [g]) + + response = await create_response( + async_data_generator( # pyright: ignore[reportArgumentType] # yields str | bytes; the proxy hands create_response the same generator + response=_aiter(_stream_chunks()), user_api_key_dict=UserAPIKeyAuth(), request_data=_request_data() + ), + media_type="text/event-stream", + headers={}, + ) + + assert response.status_code == 403 + assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret" + + async def test_streaming_buffered_holds_chunks_until_decision(): call_order: list[str] = [] From 93c5cebf9d0982634def8d67f5c6dc065ca352db Mon Sep 17 00:00:00 2001 From: r3ddyarun Date: Fri, 30 Oct 2026 00:17:38 -0500 Subject: [PATCH 9/9] fix(guardrails): answer ThirdLaw streamed Responses and Messages blocks with their block status A block decided at the end of a buffered /v1/responses stream went out as a Responses error event with a hardcoded 400, and create_response answered with that code. On /v1/messages the refusal was raw error-frame bytes, which carry no status, so the client got a 200 and a failed stream instead of the 403 ThirdLaw rendered _stream_error_items now takes the resolved block status. The Responses event carries it as its code, and the Messages refusal is an AnthropicErrorSseFrame, the frame create_response already answers as a JSON error with its own status. Its error type now follows the status (permission_error for a 403) rather than the non-standard guardrail_error. The Messages restamper used to re-encode every str chunk to bytes while armed, which dropped the frame's status, so it passes an error frame through untouched. Refusals without a verdict keep 400, as before Part of TL-2934 --- .../streaming_model_restamp.py | 8 +- .../guardrail_hooks/thirdlaw/thirdlaw.py | 35 ++++--- .../test_streaming_model_restamp.py | 13 ++- .../guardrail_hooks/test_thirdlaw.py | 94 +++++++++++++++++-- 4 files changed, 125 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py index 32e0f6e8682..688cb4250c0 100644 --- a/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py +++ b/litellm/proxy/anthropic_endpoints/streaming_model_restamp.py @@ -14,6 +14,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends _MESSAGE_START_EVENT: Final = "message_start" @@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper: def __init__(self, requested_model: str) -> None: self._requested_model: Final = requested_model - self._held = b"" + self._held: bytes = b"" self._armed = True def process(self, chunk: object) -> object: if not self._armed: return chunk + if isinstance(chunk, AnthropicErrorSseFrame) and not self._held: + # An error event carries no model, and re-encoding it would drop the status the proxy + # answers with when the stream fails before its first byte + self._armed = False + return chunk if isinstance(chunk, (bytes, bytearray)): return self._process_encoded(bytes(chunk)) if isinstance(chunk, str): diff --git a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py index 1048d0177c1..eba785cc628 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py +++ b/litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm._version import version as litellm_version +from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame from litellm.caching import DualCache from litellm.exceptions import GuardrailRaisedException from litellm.exceptions import Timeout as LitellmTimeout @@ -37,7 +38,6 @@ from litellm.proxy._types import ( ) from litellm.proxy.guardrails.anthropic_sse import ( anthropic_sse_chunks_from_body, - anthropic_sse_error_frames, assemble_anthropic_sse_body, sse_stream_text, ) @@ -1028,23 +1028,25 @@ class ThirdlawGuardrail(CustomGuardrail): @staticmethod def _stream_error_items( - message: str, surface: StreamSurface, collected: Sequence[object] + message: str, surface: StreamSurface, collected: Sequence[object], *, status_code: int ) -> Sequence[object] | None: """Terminal stream items framing a refusal in this surface's wire format. ``None`` means the surface cannot frame its own error, so the caller raises instead and - lets the proxy's data generator serialize it. + lets the proxy's data generator serialize it. ``status_code`` travels inside the frame: + when the refusal is the stream's first item the proxy answers with it as a JSON error, + and after a keepalive ping has flushed a 200 it is only the frame's body. """ match surface: case StreamSurface.ANTHROPIC_MESSAGES: - return anthropic_sse_error_frames(message) + return (anthropic_error_sse_frame(status_code, message),) case StreamSurface.RESPONSES: - return ThirdlawGuardrail._responses_error_events(message, collected) + return ThirdlawGuardrail._responses_error_events(message, collected, status_code) case StreamSurface.CHAT_COMPLETIONS | StreamSurface.OPAQUE_SSE: return None @staticmethod - def _responses_error_events(message: str, collected: Sequence[object]) -> Sequence[object]: + def _responses_error_events(message: str, collected: Sequence[object], status_code: int) -> Sequence[object]: """A ``/v1/responses`` refusal as an in-stream error event. A raised exception reaches the client as the proxy's ``{"error": ...}`` blob, which carries @@ -1059,7 +1061,7 @@ class ThirdlawGuardrail(CustomGuardrail): return ( OpenAIResponsesHandler().build_stream_error_items( - HTTPException(status_code=400, detail=message), + HTTPException(status_code=status_code, detail=message), responses_so_far=collected, ) or () @@ -1102,7 +1104,10 @@ class ThirdlawGuardrail(CustomGuardrail): except Exception as error: # noqa: BLE001 # after keepalive flush a raise cannot reach the client; send a frame flushed_frames: Final = ( self._stream_error_items( - f"ThirdLaw guardrail request failed: {_service_error_message(error)}", surface, collected + f"ThirdLaw guardrail request failed: {_service_error_message(error)}", + surface, + collected, + status_code=self._block_status(None), ) if self._sse_headers_flushed(started) else None @@ -1136,9 +1141,10 @@ class ThirdlawGuardrail(CustomGuardrail): if decision.action == "block": message: Final = decision.message or "Content violates ThirdLaw policy" - block_items: Final = self._stream_error_items(message, surface, collected) + block_status: Final = self._block_status(decision.response_status) + block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status) if block_items is None: - raise self._streaming_block_error(message, response_status=decision.response_status) + raise self._streaming_block_error(message, response_status=block_status) for item in block_items: yield item return @@ -1193,7 +1199,9 @@ class ThirdlawGuardrail(CustomGuardrail): for item in collected: yield item return - refusal_items: Final = self._stream_error_items(refusal, surface, collected) + refusal_items: Final = self._stream_error_items( + refusal, surface, collected, status_code=self._block_status(None) + ) if refusal_items is None: raise self._streaming_block_error(refusal) for item in refusal_items: @@ -1323,9 +1331,10 @@ class ThirdlawGuardrail(CustomGuardrail): return if final_decision.action == "block": message: Final = final_decision.message or "Content violates ThirdLaw policy" - block_items: Final = self._stream_error_items(message, surface, collected) + block_status: Final = self._block_status(final_decision.response_status) + block_items: Final = self._stream_error_items(message, surface, collected, status_code=block_status) if block_items is None: - raise self._streaming_block_error(message, response_status=final_decision.response_status) + raise self._streaming_block_error(message, response_status=block_status) for item in block_items: yield item return diff --git a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py index 5911fcdc073..e1fe3ce5ee5 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py +++ b/tests/unit/proxy/anthropic_endpoints/test_streaming_model_restamp.py @@ -8,6 +8,7 @@ from typing import Final import pytest +from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame from litellm.caching.caching import DualCache from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.anthropic_endpoints.streaming_model_restamp import ( @@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str: def test_restamps_sse_bytes_frame(): - restamped = restamp_anthropic_stream_chunk_model( - _message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1" - ) + restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1") assert isinstance(restamped, bytes) assert _model_from_frame(restamped) == "claude-auto-1" @@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper(): assert restamper.process(late_message_start) == late_message_start +def test_an_error_frame_passes_through_with_its_status(): + """A stream that fails before its first byte is answered with the frame's status, which only + the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200.""" + frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked") + + assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame + + def test_oversized_unterminated_chunk_flushes_unmodified(): blob = b"data: " + b"x" * 70000 restamper = AnthropicStreamModelRestamper("claude-auto-1") diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py index 4819b139f70..c7da867c5dc 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py @@ -1243,6 +1243,11 @@ async def _collect(agen: AsyncIterator[object]) -> list[object]: return [item async for item in agen] +def _sse_text(out: Sequence[object]) -> str: + """Emitted SSE frames as text. Provider frames are bytes and a refusal frame is a str.""" + return "".join(item.decode() if isinstance(item, bytes) else item for item in out if isinstance(item, (bytes, str))) + + class _StreamWithHiddenParams: """Mimics CustomStreamWrapper: an async-iterable whose chunks are plain ModelResponseStream objects, but which itself carries _hidden_params (the provider response headers, snapshotted @@ -1476,6 +1481,73 @@ async def test_a_buffered_streamed_block_reaches_the_client_as_its_block_status( assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret" +async def test_a_buffered_responses_stream_block_reaches_the_client_as_its_block_status( + monkeypatch: pytest.MonkeyPatch, +): + """The Responses error event carried a fixed 400, which the proxy answered with (TL-2934).""" + import litellm + from litellm.proxy.common_request_processing import create_response + from litellm.proxy.proxy_server import ( + async_data_generator, # pyright: ignore[reportUnknownVariableType] # its response parameter is untyped + ) + + g = _make_guardrail( + event_hook="post_call", + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})], + ) + monkeypatch.setattr(litellm, "callbacks", [g]) + + response = await create_response( + async_data_generator( # pyright: ignore[reportArgumentType] # yields str | bytes; the proxy hands create_response the same generator + response=_aiter(_responses_stream_chunks()), + user_api_key_dict=UserAPIKeyAuth(), + request_data=_request_data(), + responses_stream_errors=True, + ), + media_type="text/event-stream", + headers={}, + ) + + assert response.status_code == 403 + assert json.loads(bytes(response.body))["error"]["message"] == "leaked secret" + + +async def test_a_buffered_messages_stream_block_reaches_the_client_as_its_block_status( + monkeypatch: pytest.MonkeyPatch, +): + """A /v1/messages block was a raw error frame under a 200, so an Anthropic client saw a + successful call that failed mid-stream (TL-2934). ``restamp_model`` is set the way the proxy + sets it for every /v1/messages stream.""" + import litellm + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing, create_response + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.utils import ProxyLogging + + g = _make_guardrail( + event_hook="post_call", + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})], + ) + monkeypatch.setattr(litellm, "callbacks", [g]) + + response = await create_response( + ProxyBaseLLMRequestProcessing.async_sse_data_generator( # pyright: ignore[reportUnknownMemberType] # its request_data parameter is an untyped dict + response=_aiter(_anthropic_sse_frames()), + user_api_key_dict=UserAPIKeyAuth(), + request_data=_request_data(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()), + restamp_model="claude-alias", + ), + media_type="text/event-stream", + headers={}, + ) + + assert response.status_code == 403 + assert json.loads(bytes(response.body)) == { + "type": "error", + "error": {"type": "permission_error", "message": "leaked secret"}, + } + + async def test_streaming_buffered_holds_chunks_until_decision(): call_order: list[str] = [] @@ -1591,9 +1663,9 @@ async def test_a_truncated_tool_argument_can_still_be_blocked(): request_data=_request_data(), ) ) - blob = b"".join(item for item in out if isinstance(item, bytes)) - assert b"secret in tool args" in blob - assert b"sk-live-TRUNC" not in blob + blob = _sse_text(out) + assert "secret in tool args" in blob + assert "sk-live-TRUNC" not in blob def _anthropic_sse_frames() -> list[bytes]: @@ -1806,7 +1878,11 @@ async def test_a_chat_shaped_rewrite_of_a_messages_stream_fails_closed(): async def test_streaming_raw_sse_block_emits_anthropic_error_frame(): - g = _make_guardrail(decisions=[_decision_response({"action": "block", "message": "leaked secret"})]) + from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame + + g = _make_guardrail( + decisions=[_decision_response({"action": "block", "message": "leaked secret", "response_status": 403})] + ) out = await _collect( g.async_post_call_streaming_iterator_hook( user_api_key_dict=UserAPIKeyAuth(), @@ -1815,9 +1891,11 @@ async def test_streaming_raw_sse_block_emits_anthropic_error_frame(): ) ) assert len(out) == 1 - assert isinstance(out[0], bytes) - assert b"event: error" in out[0] - assert b"leaked secret" in out[0] + frame = out[0] + assert isinstance(frame, AnthropicErrorSseFrame) + assert frame.status_code == 403 + assert frame.startswith("event: error\n") + assert frame.error_response["error"] == {"type": "permission_error", "message": "leaked secret"} def _unscannable_chunks() -> list[dict]: @@ -2539,7 +2617,7 @@ async def test_a_buffered_messages_stream_gets_an_error_frame_when_the_service_f user_api_key_dict=UserAPIKeyAuth(), response=_aiter(_anthropic_sse_frames()), request_data=_request_data() ) ) - frames = b"".join(item for item in out if isinstance(item, bytes)).decode() + frames = _sse_text(out) assert "event: error" in frames assert "ThirdLaw guardrail request failed" in frames assert "hello" not in frames