This commit is contained in:
r3ddyarun 2026-10-30 09:41:42 -05:00 • committed by GitHub
commit c466fecae1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 5070 additions and 9 deletions

View file

@ -14,6 +14,7 @@ from typing import Final
from pydantic import TypeAdapter, ValidationError
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
_MESSAGE_START_EVENT: Final = "message_start"
@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper:
def __init__(self, requested_model: str) -> None:
self._requested_model: Final = requested_model
self._held = b""
self._held: bytes = b""
self._armed = True
def process(self, chunk: object) -> object:
if not self._armed:
return chunk
if isinstance(chunk, AnthropicErrorSseFrame) and not self._held:
# An error event carries no model, and re-encoding it would drop the status the proxy
# answers with when the stream fails before its first byte
self._armed = False
return chunk
if isinstance(chunk, (bytes, bytearray)):
return self._process_encoded(bytes(chunk))
if isinstance(chunk, str):

View file

@ -9,12 +9,24 @@ from __future__ import annotations
import json
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from functools import reduce
from types import MappingProxyType
from typing import Final
from typing import (
TYPE_CHECKING,
Final,
cast, # noqa: TID251 # the Messages body is a total=False TypedDict; validating through it drops provider fields
)
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.messages.stream_assembly import (
_joined_sse_stream, # pyright: ignore[reportPrivateUsage] # moved with the assembler; a copy splits trailing utf-8
_parsed_sse_events, # pyright: ignore[reportPrivateUsage] # same parser assemble_anthropic_sse_stream uses
anthropic_message_delta_fields,
)
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
@ -26,6 +38,20 @@ from litellm.llms.anthropic.messages.stream_assembly import (
)
from litellm.types.utils import Choices, ModelResponse
if TYPE_CHECKING:
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
_EMPTY_MAP: Final[Mapping[str, object]] = MappingProxyType({})
_TOOL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"})
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
def sse_stream_text(all_chunks: Sequence[object]) -> str | None:
"""The raw SSE frames joined as text, for a caller that forwards the stream instead of folding it."""
return _joined_sse_stream(all_chunks)
def model_response_text(response: ModelResponse) -> str:
"""Assistant text of a response, used to detect whether a guardrail rewrote it."""
@ -137,3 +163,162 @@ def anthropic_sse_chunks_from_response(
),
).chunks
)
def _str_field(source: Mapping[str, object], key: str) -> str:
return value if isinstance(value := source.get(key), str) else ""
def _object_field(source: Mapping[str, object], key: str) -> Mapping[str, object] | None:
value: Final = source.get(key)
return _JSON_OBJECT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else None
def _accumulated_tool_input(joined: str) -> object:
"""Assemble a tool block's streamed arguments.
A stream cut mid-block leaves unparseable JSON. Returning an empty input there would
drop whatever the model had already emitted, and a caller can force exactly that by
prompting for restricted content inside a tool argument and letting max_tokens truncate
it: the text would reach the client in the replayed frames having never been scanned.
So the raw fragment is carried under ``_raw`` instead, keeping the block a well-formed
object while leaving the text visible to whatever inspects the assembled body.
"""
try:
return json.loads(joined)
except json.JSONDecodeError:
return {"_raw": joined}
@dataclass(frozen=True, slots=True)
class _ContentBlock:
"""A ``content_block_start`` and every delta that has landed on it since."""
start: Mapping[str, object]
text: tuple[str, ...] = ()
thinking: tuple[str, ...] = ()
signature: tuple[str, ...] = ()
partial_json: tuple[str, ...] = ()
citations: tuple[object, ...] = ()
def with_delta(self, delta: Mapping[str, object]) -> _ContentBlock:
match delta.get("type"):
case "text_delta":
return replace(self, text=(*self.text, _str_field(delta, "text")))
case "thinking_delta":
return replace(self, thinking=(*self.thinking, _str_field(delta, "thinking")))
case "signature_delta":
return replace(self, signature=(*self.signature, _str_field(delta, "signature")))
case "input_json_delta":
return replace(self, partial_json=(*self.partial_json, _str_field(delta, "partial_json")))
case "citations_delta":
citation: Final = delta.get("citation")
return self if citation is None else replace(self, citations=(*self.citations, citation))
case _:
return self
def _accumulated_fields(self) -> Mapping[str, object]:
block_type: Final = self.start.get("type")
if block_type == "text":
return {"text": "".join(self.text)}
if block_type == "thinking":
return {"thinking": "".join(self.thinking), "signature": "".join(self.signature)}
if block_type in _TOOL_BLOCK_TYPES:
joined: Final = "".join(self.partial_json)
return {"input": _accumulated_tool_input(joined)} if joined else _EMPTY_MAP
return _EMPTY_MAP
def rendered(self) -> Mapping[str, object]:
cited: Final[Mapping[str, object]] = (
MappingProxyType({"citations": tuple(self.citations)}) if self.citations else _EMPTY_MAP
)
return {**self.start, **self._accumulated_fields(), **cited}
_EMPTY_BLOCKS: Final[Mapping[int, _ContentBlock]] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
class _AnthropicMessage:
envelope: Mapping[str, object] | None = None
blocks: Mapping[int, _ContentBlock] = _EMPTY_BLOCKS
stop_reason: object = None
stop_sequence: object = None
delta_usage: Mapping[str, object] = _EMPTY_MAP
def _with_block(state: _AnthropicMessage, index: int, block: _ContentBlock) -> _AnthropicMessage:
return replace(state, blocks=MappingProxyType({**state.blocks, index: block}))
def _with_message_delta(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage:
stops: Final = _object_field(event, "delta") or _EMPTY_MAP
usage: Final = _object_field(event, "usage")
return replace(
state,
stop_reason=stops.get("stop_reason", state.stop_reason),
stop_sequence=stops.get("stop_sequence", state.stop_sequence),
delta_usage=MappingProxyType({**state.delta_usage, **usage}) if usage is not None else state.delta_usage,
)
def _with_event(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage:
match event.get("type"):
case "message_start":
message: Final = _object_field(event, "message")
return state if message is None else replace(state, envelope=message)
case "content_block_start":
index: Final = event.get("index")
block: Final = _object_field(event, "content_block")
if not isinstance(index, int) or block is None:
return state
return _with_block(state, index, _ContentBlock(start=block))
case "content_block_delta":
delta_index: Final = event.get("index")
delta: Final = _object_field(event, "delta")
if not isinstance(delta_index, int) or delta is None:
return state
started: Final = state.blocks.get(delta_index)
return state if started is None else _with_block(state, delta_index, started.with_delta(delta))
case "message_delta":
return _with_message_delta(state, event)
case _:
return state
def assemble_anthropic_sse_body(all_chunks: Sequence[object]) -> Mapping[str, object] | None:
"""Fold raw Anthropic SSE frames back into the non-streaming Messages body.
``assemble_anthropic_sse_stream`` folds them into a ``ModelResponse``, which is all a guardrail
scanning text needs. A guardrail that posts the provider body itself needs the shape the
non-streaming route already sends, because the chat-completions one has no field for thinking
blocks, citations, ``stop_sequence`` or the cache token split and drops them silently.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return None
state: Final = reduce(_with_event, _parsed_sse_events(sse_stream), _AnthropicMessage())
envelope: Final = state.envelope
if envelope is None:
return None
started_usage: Final = envelope.get("usage")
return {
**envelope,
"content": tuple(block.rendered() for _, block in sorted(state.blocks.items())),
"stop_reason": state.stop_reason if state.stop_reason is not None else envelope.get("stop_reason"),
"stop_sequence": state.stop_sequence if state.stop_sequence is not None else envelope.get("stop_sequence"),
"usage": {
**(started_usage if isinstance(started_usage, Mapping) else _EMPTY_MAP),
**state.delta_usage,
},
}
def anthropic_sse_chunks_from_body(body: Mapping[str, object]) -> tuple[bytes, ...]:
"""Re-emit a native Anthropic Messages body as the SSE frames its client expects."""
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
typed: Final = cast("AnthropicMessagesResponse", dict(body)) # cast-ok: validating drops provider-only fields
return tuple(FakeAnthropicMessagesStreamIterator(response=typed).chunks)

View file

@ -0,0 +1,44 @@
from typing import TYPE_CHECKING, Final
from litellm.types.guardrails import SupportedGuardrailIntegrations
from .thirdlaw import ThirdlawGuardrail
if TYPE_CHECKING:
from litellm.types.guardrails import Guardrail, LitellmParams
def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> ThirdlawGuardrail:
import litellm
# LitellmParams' multiple-inheritance MRO resolves guardrail_timeout to a
# None default, so the 60s fallback must be applied here.
_thirdlaw_callback: Final = ThirdlawGuardrail(
api_base=litellm_params.api_base,
api_key=litellm_params.api_key,
additional_headers=getattr(litellm_params, "additional_headers", None),
guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None) or 60,
streaming_buffer_until_moderated=getattr(litellm_params, "streaming_buffer_until_moderated", True),
streaming_end_of_stream_only=getattr(litellm_params, "streaming_end_of_stream_only", True),
streaming_sampling_rate=getattr(litellm_params, "streaming_sampling_rate", 5),
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", None) or "fail_closed",
unscannable_stream_fallback=getattr(litellm_params, "unscannable_stream_fallback", None) or "fail_closed",
send_stream_chunks=getattr(litellm_params, "send_stream_chunks", None) is True,
additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", None),
headers=getattr(litellm_params, "headers", None),
extra_headers=getattr(litellm_params, "extra_headers", None),
guardrail_name=guardrail["guardrail_name"],
event_hook=litellm_params.mode,
default_on=litellm_params.default_on,
)
litellm.logging_callback_manager.add_litellm_callback(_thirdlaw_callback)
return _thirdlaw_callback
guardrail_initializer_registry: Final = {
SupportedGuardrailIntegrations.THIRDLAW.value: initialize_guardrail,
}
guardrail_class_registry: Final = {
SupportedGuardrailIntegrations.THIRDLAW.value: ThirdlawGuardrail,
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,155 @@
"""Wire format of a buffered streaming response, for guardrails that scan whole streams.
A `post_call` guardrail receives a stream as whatever its route produces: chat-completions chunk
objects, raw Anthropic SSE bytes, Responses API event models, or opaque SSE from a route that
frames its own. Each has to be read, refused and rewritten in its own shape, so a guardrail that
branches on a single "is this raw SSE" flag silently treats three of those four as the fourth.
Classification lives here rather than in each guardrail so the surfaces stay one list.
"""
from __future__ import annotations
import json
from collections.abc import Sequence
from enum import Enum, auto
from typing import Final
from pydantic import TypeAdapter
from litellm.proxy.guardrails.anthropic_sse import (
is_anthropic_sse_stream,
is_raw_sse_stream,
is_sse_error_stream,
)
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamEvents
_RESPONSES_EVENT_TYPE_PREFIX: Final = "response."
_OBJECT_MAP_ADAPTER: Final = TypeAdapter(dict[object, object])
# Only these carry the finished output; response.created also carries a body, but an empty one
_RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset(
{
ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
}
)
# Every event whose ``delta`` is model output already on its way to the client. Read off the event
# enum rather than listed, so an event added there cannot quietly fall out of the comparison
_RESPONSES_DELTA_EVENT_TYPES: Final = frozenset(
event.value for event in ResponsesAPIStreamEvents if event.value.endswith(".delta")
)
# What makes two delta events part of the same field of the turn, rather than two fields that
# merely streamed next to each other
_RESPONSES_DELTA_FIELD_ATTRS: Final = ("type", "item_id", "output_index", "content_index", "summary_index")
class StreamSurface(Enum):
"""Which endpoint shape a buffered stream belongs to."""
CHAT_COMPLETIONS = auto()
ANTHROPIC_MESSAGES = auto()
RESPONSES = auto()
OPAQUE_SSE = auto()
def _stream_item_field(item: object, field: str) -> object | None:
return (
_OBJECT_MAP_ADAPTER.validate_python(item).get(field) if isinstance(item, dict) else getattr(item, field, None)
)
def _stream_item_type(item: object) -> str | None:
event_type: Final = _stream_item_field(item, "type")
return event_type if isinstance(event_type, str) else None
def classify_stream(all_chunks: Sequence[object]) -> StreamSurface:
"""Wire format the buffered chunks belong to.
Raw bytes are not enough to name the surface: ``/v1/messages`` frames its own SSE, but so does
the Google ``:streamGenerateContent`` route, and reading the latter as Anthropic would refuse it
in a format its client cannot parse.
"""
if is_raw_sse_stream(all_chunks):
return StreamSurface.ANTHROPIC_MESSAGES if is_anthropic_sse_stream(all_chunks) else StreamSurface.OPAQUE_SSE
if any(
(event_type := _stream_item_type(chunk)) is not None and event_type.startswith(_RESPONSES_EVENT_TYPE_PREFIX)
for chunk in all_chunks
):
return StreamSurface.RESPONSES
return StreamSurface.CHAT_COMPLETIONS
def final_responses_api_response(all_chunks: Sequence[object]) -> ResponsesAPIResponse | None:
"""The finished body a terminal ``/v1/responses`` event carries.
Read with ``getattr`` rather than ``model_dump``: litellm's chat-to-Responses bridge stamps
``sequence_number`` straight onto ``__dict__``, which pydantic's dump path drops.
A stream cut short before it completes has to read as unassembled rather than as a clean empty
response, since ``response.created`` carries a body too and scanning that would release every
buffered delta unscanned.
"""
return next(
(
body
for chunk in reversed(all_chunks)
if _stream_item_field(chunk, "type") in _RESPONSES_TERMINAL_EVENT_TYPES
and isinstance(body := _stream_item_field(chunk, "response"), ResponsesAPIResponse)
),
None,
)
def _responses_delta_field(chunk: object) -> tuple[str, ...]:
return tuple(str(_stream_item_field(chunk, attr)) for attr in _RESPONSES_DELTA_FIELD_ATTRS)
def responses_delta_field_texts(all_chunks: Sequence[object]) -> tuple[str, ...]:
"""Text each field of a ``/v1/responses`` turn spelled out in its delta events.
One field's deltas join as they streamed, since a finding can be split across them, and
separate fields stay apart, so a reasoning summary running into the visible answer cannot
spell out text that neither of them carries on its own.
"""
deltas: Final = tuple(
(_responses_delta_field(chunk), delta)
for chunk in all_chunks
if _stream_item_field(chunk, "type") in _RESPONSES_DELTA_EVENT_TYPES
and isinstance(delta := _stream_item_field(chunk, "delta"), str)
)
return tuple(
"".join(delta for field, delta in deltas if field == streamed_field)
for streamed_field in dict.fromkeys(field for field, _ in deltas)
)
def responses_deltas_absent_from_body(all_chunks: Sequence[object], body: ResponsesAPIResponse) -> tuple[str, ...]:
"""Streamed delta text the terminal body does not account for.
Reasoning summaries and tool-call arguments reach the client through delta events that some
providers never repeat in the finished body, so a scan of that body alone would not have seen
them. Comparison runs over the JSON-escaped body so text carrying quotes or newlines is not
reported missing purely because of escaping.
"""
encoded_body: Final = json.dumps(body.model_dump(mode="json"), default=str)
return tuple(
text for text in responses_delta_field_texts(all_chunks) if text and json.dumps(text)[1:-1] not in encoded_body
)
def is_terminal_error_stream(all_chunks: Sequence[object]) -> bool:
"""Whether the buffered stream is only the refusal an earlier guardrail in the chain emitted.
`post_call` guardrails are composed, so a hook can be handed the terminal error items a
preceding one produced. Those carry no content to scan, and replacing them would hide the
refusal the client is owed.
"""
if all_chunks and all(_stream_item_type(chunk) == "error" for chunk in all_chunks):
return True
return is_sse_error_stream(all_chunks)

View file

@ -4950,7 +4950,12 @@ async def _run_background_health_check():
class StreamingCallbackError(Exception):
pass
status_code: int
def __init__(self, message: str, *, status_code: int | None = None) -> None:
super().__init__(message)
if status_code is not None:
self.status_code = status_code
# Fields in ``litellm_settings`` / ``general_settings`` whose values flow

View file

@ -68,6 +68,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.repelloai import (
from litellm.types.proxy.guardrails.guardrail_hooks.singulr import (
SingulrGuardrailConfigModel,
)
from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import (
ThirdlawGuardrailConfigModel,
)
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
ToolPermissionGuardrailConfigModel,
)
@ -152,6 +155,7 @@ class SupportedGuardrailIntegrations(Enum):
REPELLOAI = "repelloai"
SINGULR = "singulr"
HEADROOM = "headroom"
THIRDLAW = "thirdlaw"
COMPRESR = "compresr"
TYPESAFE = "typesafe"
STRAIKER = "straiker"
@ -1408,6 +1412,7 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o
HiddenlayerGuardrailConfigModel,
QostodianNexusConfigModel,
VigilGuardGuardrailConfigModel,
ThirdlawGuardrailConfigModel,
SingulrGuardrailConfigModel,
RecoConfigModel,
Agent365GuardrailConfigModel,

View file

@ -0,0 +1,132 @@
from collections.abc import Mapping, Sequence
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
class ThirdlawGuardrailRequestMetadata(BaseModel):
"""LiteLLM context for one guardrail evaluation, sent to the ThirdLaw intervene-service."""
model_config = ConfigDict(frozen=True, extra="allow")
litellm_version: str
litellm_call_id: str | None = None
litellm_trace_id: str | None = None
model: str | None = None
user_api_key_hash: str | None = None
user_api_key_alias: str | None = None
user_api_key_user_id: str | None = None
user_api_key_user_email: str | None = None
user_api_key_team_id: str | None = None
user_api_key_team_alias: str | None = None
user_api_key_end_user_id: str | None = None
user_api_key_org_id: str | None = None
user_api_key_project_id: str | None = None
user_api_key_project_alias: str | None = None
user_api_key_org_alias: str | None = None
agent_id: str | None = None
litellm_session_id: str | None = None
tags: tuple[str, ...] | None = None
user_api_key_auth_metadata: Mapping[str, object] | None = None
project_metadata: Mapping[str, object] | None = None
organization_metadata: Mapping[str, object] | None = None
class ThirdlawGuardrailRequest(BaseModel):
"""POST body for the intervene-service full-payload guardrail endpoint (/guardrails/litellm/v2)."""
model_config = ConfigDict(frozen=True)
event_type: Literal["pre_call", "during_call", "post_call"]
metadata: ThirdlawGuardrailRequestMetadata
request_url: str | None = None
request_headers: Mapping[str, str] | None = None
request_body: Mapping[str, object] | None = None
response_body: Mapping[str, object] | None = None
response_headers: Mapping[str, str] | None = None
# The buffered stream beside the assembled body: the events a /v1/responses or chat completions
# client received, or the raw SSE text a /v1/messages client received. The service may fold
# either side and fall back to the other, without a LiteLLM release in between.
response_chunks: Sequence[Mapping[str, object]] | None = None
response_sse: str | None = None
# Text a /v1/responses stream delivered in delta events that its terminal body does not repeat
# (reasoning summaries, tool-call arguments on some providers). Present only when non-empty.
streamed_deltas_not_in_body: Sequence[str] | None = None
additional_provider_specific_params: Mapping[str, object] | None = None
class ThirdlawGuardrailResponse(BaseModel):
"""Decision returned by the intervene-service.
``request_body`` / ``response_body`` are full replacements with shallow top-level
overlay semantics: a top-level key that is present is authoritative and must be
complete (e.g. a replaced ``choices`` list must carry every choice); omitted
top-level keys keep their original values.
"""
model_config = ConfigDict(frozen=True, extra="allow")
action: Literal["allow", "block", "modify_request", "modify_response"]
request_body: Mapping[str, object] | None = None
response_body: Mapping[str, object] | None = None
response_status: int | None = None
message: str = ""
class ThirdlawGuardrailConfigModelOptionalParams(BaseModel):
model_config = ConfigDict(frozen=True)
additional_headers: str | None = Field(
default=None,
description="Comma-separated list of inbound request header names whose RAW values ThirdLaw should receive. Every inbound header is always forwarded; credential headers (authorization, x-api-key, cookie) are forwarded as ***REDACTED*** unless listed here. Example: authorization,x-request-id.",
)
unreachable_fallback: Literal["fail_closed", "fail_open"] = Field(
default="fail_closed",
description="Controls LiteLLM behavior when ThirdLaw is unreachable (network error, timeout, 502-504). fail_closed blocks the request. fail_open allows the request to continue. A block decision from ThirdLaw always blocks regardless of this setting.",
)
unscannable_stream_fallback: Literal["fail_closed", "fail_open"] = Field(
default="fail_closed",
description="Controls LiteLLM behavior when a streamed response cannot be assembled into a scannable shape (for example /v1/responses and text-completion streams). fail_closed refuses the stream. fail_open forwards it unscanned, which lets a caller pick such an endpoint to bypass response moderation.",
)
send_stream_chunks: bool | None = Field(
default=False,
description="If true, a finished streamed response is posted both as the assembled provider body in response_body and as the buffered stream beside it: response_chunks for /v1/responses and chat completions, response_sse for /v1/messages. Set to false to post the assembled body alone.",
)
streaming_buffer_until_moderated: bool | None = Field(
default=True,
description="If true (default), a streamed response is withheld until ThirdLaw has moderated the assembled response; a modify_response decision is then applied before anything reaches the client.",
)
streaming_end_of_stream_only: bool | None = Field(
default=True,
description="If true (default), ThirdLaw is only called once, when a streamed response finishes, instead of periodically during streaming. Set to false to enable interim block-only checks via streaming_sampling_rate.",
)
streaming_sampling_rate: int | None = Field(
default=5,
description="When streaming_end_of_stream_only is false, check every Nth streamed chunk (in addition to the final end-of-stream check). Interim checks can only block, not modify. Ignored when streaming_end_of_stream_only is true.",
)
api_key: str | None = Field(
default=None,
description="API key for ThirdLaw, sent as a bearer token. Env: THIRDLAW_API_KEY.",
)
class ThirdlawGuardrailConfigModel(GuardrailConfigModel[ThirdlawGuardrailConfigModelOptionalParams]):
api_base: str | None = Field(
default=None,
description="ThirdLaw intervene-service base URL; /guardrails/litellm/v2 is appended unless already present. Env: THIRDLAW_API_BASE.",
json_schema_extra={
"examples": [
"https://api.thirdlaw.<your-domain>",
]
},
)
guardrail_timeout: int | None = Field(
default=60,
description="Timeout for the ThirdLaw API request. In seconds.",
)
@staticmethod
def ui_friendly_name() -> str:
return "ThirdLaw"

View file

@ -59,8 +59,20 @@ EXCLUDED_TERMINAL_VARS = {
"ALACRITTY_SOCKET",
}
# Guardrail integrations whose settings docs live in the separate
# BerriAI/litellm-docs repository, so they cannot be added in the same commit
# that introduces the integration here. Drop the entry once the docs land.
EXCLUDED_PENDING_DOCS_VARS = {
"THIRDLAW_API_BASE",
"THIRDLAW_API_KEY",
}
EXCLUDED_KEYS = frozenset(
EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS | EXCLUDED_INTERNAL_TUNING_VARS
EXCLUDED_TERMINAL_VARS
| EXCLUDED_GUARD_ONLY_VARS
| EXCLUDED_ROLLOUT_FLAGS
| EXCLUDED_INTERNAL_TUNING_VARS
| EXCLUDED_PENDING_DOCS_VARS
)
# Directories to skip (dependencies, venvs, caches) - only scan litellm source

View file

@ -8,6 +8,7 @@ from typing import Final
import pytest
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
from litellm.caching.caching import DualCache
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str:
def test_restamps_sse_bytes_frame():
restamped = restamp_anthropic_stream_chunk_model(
_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1"
)
restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1")
assert isinstance(restamped, bytes)
assert _model_from_frame(restamped) == "claude-auto-1"
@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper():
assert restamper.process(late_message_start) == late_message_start
def test_an_error_frame_passes_through_with_its_status():
"""A stream that fails before its first byte is answered with the frame's status, which only
the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200."""
frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked")
assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame
def test_oversized_unterminated_chunk_flushes_unmodified():
blob = b"data: " + b"x" * 70000
restamper = AnthropicStreamModelRestamper("claude-auto-1")

File diff suppressed because it is too large Load diff

View file

@ -1,17 +1,225 @@
"""Assembly contract for raw Anthropic SSE streams.
Four guardrails (bedrock, model_armor, tool_permission, thirdlaw) and the OTel request/response
recorder all scan the body this module assembles, so anything it drops is invisible to every one
of them.
"""
import json
from collections.abc import Callable
from collections.abc import Callable, Sequence
from itertools import chain
from typing import Final
import pytest
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_body,
anthropic_sse_chunks_from_response,
assemble_anthropic_sse_body,
assemble_anthropic_sse_stream,
)
from litellm.types.utils import Choices, ModelResponse
def _frames(*events: tuple[str, dict[str, object]]) -> list[bytes]:
return [f"event: {name}\ndata: {json.dumps(body)}\n\n".encode() for name, body in events]
def _message_start() -> tuple[str, dict[str, object]]:
return (
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 3, "output_tokens": 0},
},
},
)
def _tool_block_start(index: int = 0) -> tuple[str, dict[str, object]]:
return (
"content_block_start",
{
"type": "content_block_start",
"index": index,
"content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}},
},
)
def _input_delta(partial_json: str, index: int = 0) -> tuple[str, dict[str, object]]:
return (
"content_block_delta",
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "input_json_delta", "partial_json": partial_json},
},
)
def _assembled_tool_input(*events: tuple[str, dict[str, object]]) -> object:
"""The tool block as it lands in the native Anthropic body, which is the shape the /v1/messages
scan is handed. The ModelResponse conversion keeps the raw partial JSON by another route, so
asserting through that one would pass either way."""
body = assemble_anthropic_sse_body(_frames(*events))
assert body is not None
content = body["content"]
assert isinstance(content, Sequence)
return content[0]["input"]
def test_complete_tool_arguments_assemble_as_the_object_the_model_sent():
arguments = _assembled_tool_input(
_message_start(),
_tool_block_start(),
_input_delta('{"query": "weather'),
_input_delta(' in sf"}'),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
("message_stop", {"type": "message_stop"}),
)
assert arguments == {"query": "weather in sf"}
def test_truncated_tool_arguments_keep_the_text_instead_of_dropping_it():
"""max_tokens can cut a tool argument mid-JSON. That fragment still reached the client, so
discarding it here would hand a caller a way to smuggle text past every scanner above."""
arguments = _assembled_tool_input(
_message_start(),
_tool_block_start(),
_input_delta('{"query": "sk-live-TRUNC'),
(
"message_delta",
{"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}},
),
("message_stop", {"type": "message_stop"}),
)
assert "sk-live-TRUNC" in json.dumps(arguments)
def test_a_tool_block_with_no_arguments_assembles_as_an_empty_object():
"""A tool called with no arguments is not truncation, so it must not pick up a raw fragment."""
arguments = _assembled_tool_input(
_message_start(),
_tool_block_start(),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
("message_stop", {"type": "message_stop"}),
)
assert arguments == {}
def _text_block_start(index: int = 0) -> tuple[str, dict[str, object]]:
return (
"content_block_start",
{"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}},
)
def _block_delta(delta: dict[str, object], index: int = 0) -> tuple[str, dict[str, object]]:
return ("content_block_delta", {"type": "content_block_delta", "index": index, "delta": delta})
def _assembled_content(*events: tuple[str, dict[str, object]]) -> Sequence[dict[str, object]]:
body = assemble_anthropic_sse_body(_frames(*events))
assert body is not None
content = body["content"]
assert isinstance(content, Sequence)
return content
def test_citation_deltas_land_on_the_text_block_they_annotate():
citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0}
content = _assembled_content(
_message_start(),
_text_block_start(),
_block_delta({"type": "text_delta", "text": "Blue."}),
_block_delta({"type": "citations_delta", "citation": citation}),
_block_delta({"type": "citations_delta"}),
_block_delta({"type": "some_future_delta", "text": "ignored"}),
)
assert list(content) == [{"type": "text", "text": "Blue.", "citations": (citation,)}]
def test_a_block_type_with_no_deltas_keeps_its_start_payload():
redacted = {"type": "redacted_thinking", "data": "opaque-blob"}
content = _assembled_content(
_message_start(),
("content_block_start", {"type": "content_block_start", "index": 0, "content_block": redacted}),
)
assert list(content) == [redacted]
def test_malformed_block_events_are_skipped_without_corrupting_the_rest():
content = _assembled_content(
_message_start(),
("content_block_start", {"type": "content_block_start", "content_block": {"type": "text", "text": ""}}),
_text_block_start(),
("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": "not-a-mapping"}),
_block_delta({"type": "text_delta", "text": "kept"}),
)
assert list(content) == [{"type": "text", "text": "kept"}]
def test_an_undecodable_stream_assembles_to_nothing():
"""Nothing scannable comes out of bytes that are not UTF-8, so callers must treat the stream as
unassembled rather than scanning a partial body."""
frames = _frames(_message_start(), _text_block_start())
assert assemble_anthropic_sse_body([*frames[:1], b"\xff\xfe\xfd", *frames[1:]]) is None
def test_a_replayed_body_keeps_its_citations_and_redacted_thinking():
"""A guardrail rewrite is replayed to the client from the body, so anything the replay drops is
lost to the caller even though the scan saw it: citations lose their sources, and a
redacted_thinking block without its data cannot be sent back on the next turn."""
citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0}
content = [
{"type": "redacted_thinking", "data": "opaque-blob"},
{"type": "text", "text": "The sky is blue.", "citations": [citation]},
{"type": "text", "text": "No source here."},
]
body: dict[str, object] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": content,
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 3, "output_tokens": 7},
}
replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body)))
assert replayed is not None
assert json.loads(json.dumps(replayed["content"])) == content
def test_a_tool_input_sent_whole_in_its_block_start_survives_assembly():
"""Server tool blocks reach the stream with their full input in ``content_block_start`` and no
argument deltas. Blanking that input would replay a query to the client that was never scanned."""
body: dict[str, object] = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5",
"content": [
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "restricted"}}
],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 3, "output_tokens": 7},
}
replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body)))
assert replayed is not None
assert json.loads(json.dumps(replayed["content"])) == body["content"]
def _frame(event: str, data: dict[str, object]) -> bytes:
return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()

View file

@ -0,0 +1,122 @@
import json
import pytest
from litellm.proxy.guardrails.stream_surface import (
StreamSurface,
classify_stream,
is_terminal_error_stream,
)
from litellm.types.llms.openai import (
ErrorEvent,
ErrorEventError,
OutputTextDeltaEvent,
ResponseCompletedEvent,
ResponsesAPIResponse,
ResponsesAPIStreamEvents,
)
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def _chat_chunks() -> tuple[ModelResponseStream, ...]:
return (
ModelResponseStream(
model="gpt-4o-mini",
choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="hi"))],
),
)
def _anthropic_sse_frames() -> tuple[bytes, ...]:
message_start = {
"type": "message_start",
"message": {"id": "msg_1", "model": "claude-haiku-4-5", "content": [], "usage": {"input_tokens": 3}},
}
delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}
return (
f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode(),
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode(),
)
def _google_sse_frames() -> tuple[bytes, ...]:
"""The :streamGenerateContent route marks its stream raw too, but is not Anthropic."""
payload = {"candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}]}
return (f"data: {json.dumps(payload)}\n\n".encode(),)
def _responses_events() -> tuple[object, ...]:
return (
OutputTextDeltaEvent(
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
item_id="msg_1",
output_index=0,
content_index=0,
delta="hi",
),
ResponseCompletedEvent(
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
response=ResponsesAPIResponse(
id="resp_1",
created_at=1,
model="claude-haiku-4-5",
object="response",
output=[],
parallel_tool_calls=True,
tool_choice="auto",
tools=[],
top_p=1.0,
),
),
)
@pytest.mark.parametrize(
"chunks, expected",
[
(_chat_chunks(), StreamSurface.CHAT_COMPLETIONS),
(_anthropic_sse_frames(), StreamSurface.ANTHROPIC_MESSAGES),
(_responses_events(), StreamSurface.RESPONSES),
(_google_sse_frames(), StreamSurface.OPAQUE_SSE),
((), StreamSurface.CHAT_COMPLETIONS),
],
)
def test_classify_stream_names_each_wire_format(chunks, expected):
assert classify_stream(chunks) is expected
def test_opaque_sse_is_not_mistaken_for_anthropic():
"""Reading a Google stream as Anthropic would refuse it in a format its client cannot parse."""
assert classify_stream(_google_sse_frames()) is not StreamSurface.ANTHROPIC_MESSAGES
def test_responses_events_are_classified_from_a_plain_dict_too():
"""Some producers hand the hook already-serialized events rather than the pydantic models."""
assert classify_stream(({"type": "response.output_text.delta", "delta": "hi"},)) is StreamSurface.RESPONSES
def test_a_responses_error_only_stream_is_a_terminal_error_stream():
events = (
ErrorEvent(
type=ResponsesAPIStreamEvents.ERROR,
sequence_number=1,
error=ErrorEventError(type="guardrail_error", code="400", message="blocked upstream"),
),
)
assert is_terminal_error_stream(events) is True
def test_an_anthropic_error_only_stream_is_a_terminal_error_stream():
frames = (b'event: error\ndata: {"type": "error", "error": {"message": "blocked upstream"}}\n\n',)
assert is_terminal_error_stream(frames) is True
def test_a_stream_carrying_content_is_not_a_terminal_error_stream():
assert is_terminal_error_stream(_responses_events()) is False
assert is_terminal_error_stream(_chat_chunks()) is False
assert is_terminal_error_stream(_anthropic_sse_frames()) is False
def test_an_empty_stream_is_not_a_terminal_error_stream():
"""all() over nothing is True, so an empty stream would otherwise read as somebody's refusal."""
assert is_terminal_error_stream(()) is False

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 9.5 KiB

View file

@ -322,6 +322,12 @@ export const GUARDRAIL_PRESETS: Record<string, GuardrailPreset> = {
mode: "pre_call",
defaultOn: false,
},
thirdlaw: {
provider: "ThirdLaw",
guardrailNameSuggestion: "ThirdLaw Guardrail",
mode: "pre_call",
defaultOn: false,
},
promptguard: {
provider: "Promptguard",
guardrailNameSuggestion: "PromptGuard",

View file

@ -33,6 +33,7 @@ const EXPECTED_PARTNER_LOGO_FILES: Record<string, string> = {
deepkeep: "deepkeep.svg",
repelloai: "repelloai.png",
straiker: "straiker.svg",
thirdlaw: "thirdlaw.svg",
alice: "alice.svg",
agent_365: "microsoft_azure.svg",
llm_shield_proxy: "llm_shield_proxy.svg",

View file

@ -507,6 +507,16 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [
tags: ["Security", "Policy", "Prompt Injection"],
providerKey: "Repelloai",
},
{
id: "thirdlaw",
name: "ThirdLaw",
description:
"Runtime policy enforcement for LiteLLM traffic. Evaluates requests and responses against centrally managed policies called Laws, then returns a decision to continue, modify, or block.",
category: "partner",
logo: guardrailLogoMap["ThirdLaw"],
tags: ["Policy Enforcement", "Runtime Security", "Intervention"],
providerKey: "ThirdLaw",
},
{
id: "straiker",
name: "Straiker",

View file

@ -224,6 +224,20 @@ describe("guardrail_info_helpers", () => {
expect(result.displayName).toBe("RepelloAI Argus");
expect(result.logo).toContain("repelloai.png");
});
it("should return thirdlaw logo when ui_friendly_name is ThirdLaw", () => {
populateGuardrailProviders({
thirdlaw: { ui_friendly_name: "ThirdLaw" },
});
populateGuardrailProviderMap({
thirdlaw: { ui_friendly_name: "ThirdLaw" },
});
const result = getGuardrailLogoAndName("thirdlaw");
expect(result.displayName).toBe("ThirdLaw");
expect(result.logo).toContain("thirdlaw.svg");
});
});
describe("formatGuardrailMode", () => {

View file

@ -29,6 +29,7 @@ import repelloAiLogo from "../../../../../public/assets/logos/repelloai.png";
import straikerLogo from "../../../../../public/assets/logos/straiker.svg";
import wingbackLogo from "../../../../../public/assets/logos/wingback.png";
import xecguardLogo from "../../../../../public/assets/logos/xecguard.svg";
import thirdlawLogo from "../../../../../public/assets/logos/thirdlaw.svg";
import zscalerLogo from "../../../../../public/assets/logos/zscaler.svg";
// Legacy enum - keeping for backward compatibility
@ -88,6 +89,7 @@ export const guardrail_provider_map: Record<string, string> = {
Deepkeep: "deepkeep",
QostodianNexus: "qostodian_nexus",
Repelloai: "repelloai",
ThirdLaw: "thirdlaw",
Alice: "alice",
"LLM Shield Proxy": "llm_shield_proxy",
Conduct: "conduct",
@ -290,6 +292,7 @@ export const shouldRenderDecisionModelFields = (provider: string | null) => {
};
export const guardrailLogoMap = {
ThirdLaw: thirdlawLogo.src,
"Zscaler AI Guard": zscalerLogo.src,
"Presidio PII": microsoftAzureLogo.src,
"Bedrock Guardrail": bedrockLogo.src,