mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge f0926be478 into 4786dd861b
This commit is contained in:
commit
c466fecae1
19 changed files with 5070 additions and 9 deletions
|
|
@ -14,6 +14,7 @@ from typing import Final
|
|||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.anthropic_interface.exceptions import AnthropicErrorSseFrame
|
||||
from litellm.llms.custom_httpx.sse_lines import split_sse_lines_keepends
|
||||
|
||||
_MESSAGE_START_EVENT: Final = "message_start"
|
||||
|
|
@ -108,12 +109,17 @@ class AnthropicStreamModelRestamper:
|
|||
|
||||
def __init__(self, requested_model: str) -> None:
|
||||
self._requested_model: Final = requested_model
|
||||
self._held = b""
|
||||
self._held: bytes = b""
|
||||
self._armed = True
|
||||
|
||||
def process(self, chunk: object) -> object:
|
||||
if not self._armed:
|
||||
return chunk
|
||||
if isinstance(chunk, AnthropicErrorSseFrame) and not self._held:
|
||||
# An error event carries no model, and re-encoding it would drop the status the proxy
|
||||
# answers with when the stream fails before its first byte
|
||||
self._armed = False
|
||||
return chunk
|
||||
if isinstance(chunk, (bytes, bytearray)):
|
||||
return self._process_encoded(bytes(chunk))
|
||||
if isinstance(chunk, str):
|
||||
|
|
|
|||
|
|
@ -9,12 +9,24 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import reduce
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Final,
|
||||
cast, # noqa: TID251 # the Messages body is a total=False TypedDict; validating through it drops provider fields
|
||||
)
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
|
||||
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
_joined_sse_stream, # pyright: ignore[reportPrivateUsage] # moved with the assembler; a copy splits trailing utf-8
|
||||
_parsed_sse_events, # pyright: ignore[reportPrivateUsage] # same parser assemble_anthropic_sse_stream uses
|
||||
anthropic_message_delta_fields,
|
||||
)
|
||||
from litellm.llms.anthropic.messages.stream_assembly import (
|
||||
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
|
|
@ -26,6 +38,20 @@ from litellm.llms.anthropic.messages.stream_assembly import (
|
|||
)
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
|
||||
|
||||
_EMPTY_MAP: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
_TOOL_BLOCK_TYPES: Final = frozenset({"tool_use", "server_tool_use", "mcp_tool_use"})
|
||||
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
def sse_stream_text(all_chunks: Sequence[object]) -> str | None:
|
||||
"""The raw SSE frames joined as text, for a caller that forwards the stream instead of folding it."""
|
||||
return _joined_sse_stream(all_chunks)
|
||||
|
||||
|
||||
def model_response_text(response: ModelResponse) -> str:
|
||||
"""Assistant text of a response, used to detect whether a guardrail rewrote it."""
|
||||
|
|
@ -137,3 +163,162 @@ def anthropic_sse_chunks_from_response(
|
|||
),
|
||||
).chunks
|
||||
)
|
||||
|
||||
|
||||
def _str_field(source: Mapping[str, object], key: str) -> str:
|
||||
return value if isinstance(value := source.get(key), str) else ""
|
||||
|
||||
|
||||
def _object_field(source: Mapping[str, object], key: str) -> Mapping[str, object] | None:
|
||||
value: Final = source.get(key)
|
||||
return _JSON_OBJECT_ADAPTER.validate_python(value) if isinstance(value, Mapping) else None
|
||||
|
||||
|
||||
def _accumulated_tool_input(joined: str) -> object:
|
||||
"""Assemble a tool block's streamed arguments.
|
||||
|
||||
A stream cut mid-block leaves unparseable JSON. Returning an empty input there would
|
||||
drop whatever the model had already emitted, and a caller can force exactly that by
|
||||
prompting for restricted content inside a tool argument and letting max_tokens truncate
|
||||
it: the text would reach the client in the replayed frames having never been scanned.
|
||||
So the raw fragment is carried under ``_raw`` instead, keeping the block a well-formed
|
||||
object while leaving the text visible to whatever inspects the assembled body.
|
||||
"""
|
||||
try:
|
||||
return json.loads(joined)
|
||||
except json.JSONDecodeError:
|
||||
return {"_raw": joined}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ContentBlock:
|
||||
"""A ``content_block_start`` and every delta that has landed on it since."""
|
||||
|
||||
start: Mapping[str, object]
|
||||
text: tuple[str, ...] = ()
|
||||
thinking: tuple[str, ...] = ()
|
||||
signature: tuple[str, ...] = ()
|
||||
partial_json: tuple[str, ...] = ()
|
||||
citations: tuple[object, ...] = ()
|
||||
|
||||
def with_delta(self, delta: Mapping[str, object]) -> _ContentBlock:
|
||||
match delta.get("type"):
|
||||
case "text_delta":
|
||||
return replace(self, text=(*self.text, _str_field(delta, "text")))
|
||||
case "thinking_delta":
|
||||
return replace(self, thinking=(*self.thinking, _str_field(delta, "thinking")))
|
||||
case "signature_delta":
|
||||
return replace(self, signature=(*self.signature, _str_field(delta, "signature")))
|
||||
case "input_json_delta":
|
||||
return replace(self, partial_json=(*self.partial_json, _str_field(delta, "partial_json")))
|
||||
case "citations_delta":
|
||||
citation: Final = delta.get("citation")
|
||||
return self if citation is None else replace(self, citations=(*self.citations, citation))
|
||||
case _:
|
||||
return self
|
||||
|
||||
def _accumulated_fields(self) -> Mapping[str, object]:
|
||||
block_type: Final = self.start.get("type")
|
||||
if block_type == "text":
|
||||
return {"text": "".join(self.text)}
|
||||
if block_type == "thinking":
|
||||
return {"thinking": "".join(self.thinking), "signature": "".join(self.signature)}
|
||||
if block_type in _TOOL_BLOCK_TYPES:
|
||||
joined: Final = "".join(self.partial_json)
|
||||
return {"input": _accumulated_tool_input(joined)} if joined else _EMPTY_MAP
|
||||
return _EMPTY_MAP
|
||||
|
||||
def rendered(self) -> Mapping[str, object]:
|
||||
cited: Final[Mapping[str, object]] = (
|
||||
MappingProxyType({"citations": tuple(self.citations)}) if self.citations else _EMPTY_MAP
|
||||
)
|
||||
return {**self.start, **self._accumulated_fields(), **cited}
|
||||
|
||||
|
||||
_EMPTY_BLOCKS: Final[Mapping[int, _ContentBlock]] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _AnthropicMessage:
|
||||
envelope: Mapping[str, object] | None = None
|
||||
blocks: Mapping[int, _ContentBlock] = _EMPTY_BLOCKS
|
||||
stop_reason: object = None
|
||||
stop_sequence: object = None
|
||||
delta_usage: Mapping[str, object] = _EMPTY_MAP
|
||||
|
||||
|
||||
def _with_block(state: _AnthropicMessage, index: int, block: _ContentBlock) -> _AnthropicMessage:
|
||||
return replace(state, blocks=MappingProxyType({**state.blocks, index: block}))
|
||||
|
||||
|
||||
def _with_message_delta(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage:
|
||||
stops: Final = _object_field(event, "delta") or _EMPTY_MAP
|
||||
usage: Final = _object_field(event, "usage")
|
||||
return replace(
|
||||
state,
|
||||
stop_reason=stops.get("stop_reason", state.stop_reason),
|
||||
stop_sequence=stops.get("stop_sequence", state.stop_sequence),
|
||||
delta_usage=MappingProxyType({**state.delta_usage, **usage}) if usage is not None else state.delta_usage,
|
||||
)
|
||||
|
||||
|
||||
def _with_event(state: _AnthropicMessage, event: Mapping[str, object]) -> _AnthropicMessage:
|
||||
match event.get("type"):
|
||||
case "message_start":
|
||||
message: Final = _object_field(event, "message")
|
||||
return state if message is None else replace(state, envelope=message)
|
||||
case "content_block_start":
|
||||
index: Final = event.get("index")
|
||||
block: Final = _object_field(event, "content_block")
|
||||
if not isinstance(index, int) or block is None:
|
||||
return state
|
||||
return _with_block(state, index, _ContentBlock(start=block))
|
||||
case "content_block_delta":
|
||||
delta_index: Final = event.get("index")
|
||||
delta: Final = _object_field(event, "delta")
|
||||
if not isinstance(delta_index, int) or delta is None:
|
||||
return state
|
||||
started: Final = state.blocks.get(delta_index)
|
||||
return state if started is None else _with_block(state, delta_index, started.with_delta(delta))
|
||||
case "message_delta":
|
||||
return _with_message_delta(state, event)
|
||||
case _:
|
||||
return state
|
||||
|
||||
|
||||
def assemble_anthropic_sse_body(all_chunks: Sequence[object]) -> Mapping[str, object] | None:
|
||||
"""Fold raw Anthropic SSE frames back into the non-streaming Messages body.
|
||||
|
||||
``assemble_anthropic_sse_stream`` folds them into a ``ModelResponse``, which is all a guardrail
|
||||
scanning text needs. A guardrail that posts the provider body itself needs the shape the
|
||||
non-streaming route already sends, because the chat-completions one has no field for thinking
|
||||
blocks, citations, ``stop_sequence`` or the cache token split and drops them silently.
|
||||
"""
|
||||
sse_stream: Final = _joined_sse_stream(all_chunks)
|
||||
if sse_stream is None:
|
||||
return None
|
||||
state: Final = reduce(_with_event, _parsed_sse_events(sse_stream), _AnthropicMessage())
|
||||
envelope: Final = state.envelope
|
||||
if envelope is None:
|
||||
return None
|
||||
started_usage: Final = envelope.get("usage")
|
||||
return {
|
||||
**envelope,
|
||||
"content": tuple(block.rendered() for _, block in sorted(state.blocks.items())),
|
||||
"stop_reason": state.stop_reason if state.stop_reason is not None else envelope.get("stop_reason"),
|
||||
"stop_sequence": state.stop_sequence if state.stop_sequence is not None else envelope.get("stop_sequence"),
|
||||
"usage": {
|
||||
**(started_usage if isinstance(started_usage, Mapping) else _EMPTY_MAP),
|
||||
**state.delta_usage,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def anthropic_sse_chunks_from_body(body: Mapping[str, object]) -> tuple[bytes, ...]:
|
||||
"""Re-emit a native Anthropic Messages body as the SSE frames its client expects."""
|
||||
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
|
||||
typed: Final = cast("AnthropicMessagesResponse", dict(body)) # cast-ok: validating drops provider-only fields
|
||||
return tuple(FakeAnthropicMessagesStreamIterator(response=typed).chunks)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.types.guardrails import SupportedGuardrailIntegrations
|
||||
|
||||
from .thirdlaw import ThirdlawGuardrail
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.guardrails import Guardrail, LitellmParams
|
||||
|
||||
|
||||
def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail") -> ThirdlawGuardrail:
|
||||
import litellm
|
||||
|
||||
# LitellmParams' multiple-inheritance MRO resolves guardrail_timeout to a
|
||||
# None default, so the 60s fallback must be applied here.
|
||||
_thirdlaw_callback: Final = ThirdlawGuardrail(
|
||||
api_base=litellm_params.api_base,
|
||||
api_key=litellm_params.api_key,
|
||||
additional_headers=getattr(litellm_params, "additional_headers", None),
|
||||
guardrail_timeout=getattr(litellm_params, "guardrail_timeout", None) or 60,
|
||||
streaming_buffer_until_moderated=getattr(litellm_params, "streaming_buffer_until_moderated", True),
|
||||
streaming_end_of_stream_only=getattr(litellm_params, "streaming_end_of_stream_only", True),
|
||||
streaming_sampling_rate=getattr(litellm_params, "streaming_sampling_rate", 5),
|
||||
unreachable_fallback=getattr(litellm_params, "unreachable_fallback", None) or "fail_closed",
|
||||
unscannable_stream_fallback=getattr(litellm_params, "unscannable_stream_fallback", None) or "fail_closed",
|
||||
send_stream_chunks=getattr(litellm_params, "send_stream_chunks", None) is True,
|
||||
additional_provider_specific_params=getattr(litellm_params, "additional_provider_specific_params", None),
|
||||
headers=getattr(litellm_params, "headers", None),
|
||||
extra_headers=getattr(litellm_params, "extra_headers", None),
|
||||
guardrail_name=guardrail["guardrail_name"],
|
||||
event_hook=litellm_params.mode,
|
||||
default_on=litellm_params.default_on,
|
||||
)
|
||||
litellm.logging_callback_manager.add_litellm_callback(_thirdlaw_callback)
|
||||
return _thirdlaw_callback
|
||||
|
||||
|
||||
guardrail_initializer_registry: Final = {
|
||||
SupportedGuardrailIntegrations.THIRDLAW.value: initialize_guardrail,
|
||||
}
|
||||
|
||||
guardrail_class_registry: Final = {
|
||||
SupportedGuardrailIntegrations.THIRDLAW.value: ThirdlawGuardrail,
|
||||
}
|
||||
1353
litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py
Normal file
1353
litellm/proxy/guardrails/guardrail_hooks/thirdlaw/thirdlaw.py
Normal file
File diff suppressed because it is too large
Load diff
155
litellm/proxy/guardrails/stream_surface.py
Normal file
155
litellm/proxy/guardrails/stream_surface.py
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
"""Wire format of a buffered streaming response, for guardrails that scan whole streams.
|
||||
|
||||
A `post_call` guardrail receives a stream as whatever its route produces: chat-completions chunk
|
||||
objects, raw Anthropic SSE bytes, Responses API event models, or opaque SSE from a route that
|
||||
frames its own. Each has to be read, refused and rewritten in its own shape, so a guardrail that
|
||||
branches on a single "is this raw SSE" flag silently treats three of those four as the fourth.
|
||||
|
||||
Classification lives here rather than in each guardrail so the surfaces stay one list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from enum import Enum, auto
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
is_anthropic_sse_stream,
|
||||
is_raw_sse_stream,
|
||||
is_sse_error_stream,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamEvents
|
||||
|
||||
_RESPONSES_EVENT_TYPE_PREFIX: Final = "response."
|
||||
|
||||
_OBJECT_MAP_ADAPTER: Final = TypeAdapter(dict[object, object])
|
||||
|
||||
# Only these carry the finished output; response.created also carries a body, but an empty one
|
||||
_RESPONSES_TERMINAL_EVENT_TYPES: Final = frozenset(
|
||||
{
|
||||
ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
|
||||
ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
|
||||
ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
|
||||
}
|
||||
)
|
||||
|
||||
# Every event whose ``delta`` is model output already on its way to the client. Read off the event
|
||||
# enum rather than listed, so an event added there cannot quietly fall out of the comparison
|
||||
_RESPONSES_DELTA_EVENT_TYPES: Final = frozenset(
|
||||
event.value for event in ResponsesAPIStreamEvents if event.value.endswith(".delta")
|
||||
)
|
||||
|
||||
# What makes two delta events part of the same field of the turn, rather than two fields that
|
||||
# merely streamed next to each other
|
||||
_RESPONSES_DELTA_FIELD_ATTRS: Final = ("type", "item_id", "output_index", "content_index", "summary_index")
|
||||
|
||||
|
||||
class StreamSurface(Enum):
|
||||
"""Which endpoint shape a buffered stream belongs to."""
|
||||
|
||||
CHAT_COMPLETIONS = auto()
|
||||
ANTHROPIC_MESSAGES = auto()
|
||||
RESPONSES = auto()
|
||||
OPAQUE_SSE = auto()
|
||||
|
||||
|
||||
def _stream_item_field(item: object, field: str) -> object | None:
|
||||
return (
|
||||
_OBJECT_MAP_ADAPTER.validate_python(item).get(field) if isinstance(item, dict) else getattr(item, field, None)
|
||||
)
|
||||
|
||||
|
||||
def _stream_item_type(item: object) -> str | None:
|
||||
event_type: Final = _stream_item_field(item, "type")
|
||||
return event_type if isinstance(event_type, str) else None
|
||||
|
||||
|
||||
def classify_stream(all_chunks: Sequence[object]) -> StreamSurface:
|
||||
"""Wire format the buffered chunks belong to.
|
||||
|
||||
Raw bytes are not enough to name the surface: ``/v1/messages`` frames its own SSE, but so does
|
||||
the Google ``:streamGenerateContent`` route, and reading the latter as Anthropic would refuse it
|
||||
in a format its client cannot parse.
|
||||
"""
|
||||
if is_raw_sse_stream(all_chunks):
|
||||
return StreamSurface.ANTHROPIC_MESSAGES if is_anthropic_sse_stream(all_chunks) else StreamSurface.OPAQUE_SSE
|
||||
if any(
|
||||
(event_type := _stream_item_type(chunk)) is not None and event_type.startswith(_RESPONSES_EVENT_TYPE_PREFIX)
|
||||
for chunk in all_chunks
|
||||
):
|
||||
return StreamSurface.RESPONSES
|
||||
return StreamSurface.CHAT_COMPLETIONS
|
||||
|
||||
|
||||
def final_responses_api_response(all_chunks: Sequence[object]) -> ResponsesAPIResponse | None:
|
||||
"""The finished body a terminal ``/v1/responses`` event carries.
|
||||
|
||||
Read with ``getattr`` rather than ``model_dump``: litellm's chat-to-Responses bridge stamps
|
||||
``sequence_number`` straight onto ``__dict__``, which pydantic's dump path drops.
|
||||
|
||||
A stream cut short before it completes has to read as unassembled rather than as a clean empty
|
||||
response, since ``response.created`` carries a body too and scanning that would release every
|
||||
buffered delta unscanned.
|
||||
"""
|
||||
return next(
|
||||
(
|
||||
body
|
||||
for chunk in reversed(all_chunks)
|
||||
if _stream_item_field(chunk, "type") in _RESPONSES_TERMINAL_EVENT_TYPES
|
||||
and isinstance(body := _stream_item_field(chunk, "response"), ResponsesAPIResponse)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _responses_delta_field(chunk: object) -> tuple[str, ...]:
|
||||
return tuple(str(_stream_item_field(chunk, attr)) for attr in _RESPONSES_DELTA_FIELD_ATTRS)
|
||||
|
||||
|
||||
def responses_delta_field_texts(all_chunks: Sequence[object]) -> tuple[str, ...]:
|
||||
"""Text each field of a ``/v1/responses`` turn spelled out in its delta events.
|
||||
|
||||
One field's deltas join as they streamed, since a finding can be split across them, and
|
||||
separate fields stay apart, so a reasoning summary running into the visible answer cannot
|
||||
spell out text that neither of them carries on its own.
|
||||
"""
|
||||
deltas: Final = tuple(
|
||||
(_responses_delta_field(chunk), delta)
|
||||
for chunk in all_chunks
|
||||
if _stream_item_field(chunk, "type") in _RESPONSES_DELTA_EVENT_TYPES
|
||||
and isinstance(delta := _stream_item_field(chunk, "delta"), str)
|
||||
)
|
||||
return tuple(
|
||||
"".join(delta for field, delta in deltas if field == streamed_field)
|
||||
for streamed_field in dict.fromkeys(field for field, _ in deltas)
|
||||
)
|
||||
|
||||
|
||||
def responses_deltas_absent_from_body(all_chunks: Sequence[object], body: ResponsesAPIResponse) -> tuple[str, ...]:
|
||||
"""Streamed delta text the terminal body does not account for.
|
||||
|
||||
Reasoning summaries and tool-call arguments reach the client through delta events that some
|
||||
providers never repeat in the finished body, so a scan of that body alone would not have seen
|
||||
them. Comparison runs over the JSON-escaped body so text carrying quotes or newlines is not
|
||||
reported missing purely because of escaping.
|
||||
"""
|
||||
encoded_body: Final = json.dumps(body.model_dump(mode="json"), default=str)
|
||||
return tuple(
|
||||
text for text in responses_delta_field_texts(all_chunks) if text and json.dumps(text)[1:-1] not in encoded_body
|
||||
)
|
||||
|
||||
|
||||
def is_terminal_error_stream(all_chunks: Sequence[object]) -> bool:
|
||||
"""Whether the buffered stream is only the refusal an earlier guardrail in the chain emitted.
|
||||
|
||||
`post_call` guardrails are composed, so a hook can be handed the terminal error items a
|
||||
preceding one produced. Those carry no content to scan, and replacing them would hide the
|
||||
refusal the client is owed.
|
||||
"""
|
||||
if all_chunks and all(_stream_item_type(chunk) == "error" for chunk in all_chunks):
|
||||
return True
|
||||
return is_sse_error_stream(all_chunks)
|
||||
|
|
@ -4950,7 +4950,12 @@ async def _run_background_health_check():
|
|||
|
||||
|
||||
class StreamingCallbackError(Exception):
|
||||
pass
|
||||
status_code: int
|
||||
|
||||
def __init__(self, message: str, *, status_code: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
if status_code is not None:
|
||||
self.status_code = status_code
|
||||
|
||||
|
||||
# Fields in ``litellm_settings`` / ``general_settings`` whose values flow
|
||||
|
|
|
|||
|
|
@ -68,6 +68,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.repelloai import (
|
|||
from litellm.types.proxy.guardrails.guardrail_hooks.singulr import (
|
||||
SingulrGuardrailConfigModel,
|
||||
)
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.thirdlaw import (
|
||||
ThirdlawGuardrailConfigModel,
|
||||
)
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
|
||||
ToolPermissionGuardrailConfigModel,
|
||||
)
|
||||
|
|
@ -152,6 +155,7 @@ class SupportedGuardrailIntegrations(Enum):
|
|||
REPELLOAI = "repelloai"
|
||||
SINGULR = "singulr"
|
||||
HEADROOM = "headroom"
|
||||
THIRDLAW = "thirdlaw"
|
||||
COMPRESR = "compresr"
|
||||
TYPESAFE = "typesafe"
|
||||
STRAIKER = "straiker"
|
||||
|
|
@ -1408,6 +1412,7 @@ class LitellmParams( # pyright: ignore[reportIncompatibleVariableOverride] # o
|
|||
HiddenlayerGuardrailConfigModel,
|
||||
QostodianNexusConfigModel,
|
||||
VigilGuardGuardrailConfigModel,
|
||||
ThirdlawGuardrailConfigModel,
|
||||
SingulrGuardrailConfigModel,
|
||||
RecoConfigModel,
|
||||
Agent365GuardrailConfigModel,
|
||||
|
|
|
|||
132
litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py
Normal file
132
litellm/types/proxy/guardrails/guardrail_hooks/thirdlaw.py
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
|
||||
|
||||
|
||||
class ThirdlawGuardrailRequestMetadata(BaseModel):
|
||||
"""LiteLLM context for one guardrail evaluation, sent to the ThirdLaw intervene-service."""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
litellm_version: str
|
||||
litellm_call_id: str | None = None
|
||||
litellm_trace_id: str | None = None
|
||||
model: str | None = None
|
||||
user_api_key_hash: str | None = None
|
||||
user_api_key_alias: str | None = None
|
||||
user_api_key_user_id: str | None = None
|
||||
user_api_key_user_email: str | None = None
|
||||
user_api_key_team_id: str | None = None
|
||||
user_api_key_team_alias: str | None = None
|
||||
user_api_key_end_user_id: str | None = None
|
||||
user_api_key_org_id: str | None = None
|
||||
user_api_key_project_id: str | None = None
|
||||
user_api_key_project_alias: str | None = None
|
||||
user_api_key_org_alias: str | None = None
|
||||
agent_id: str | None = None
|
||||
litellm_session_id: str | None = None
|
||||
tags: tuple[str, ...] | None = None
|
||||
user_api_key_auth_metadata: Mapping[str, object] | None = None
|
||||
project_metadata: Mapping[str, object] | None = None
|
||||
organization_metadata: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class ThirdlawGuardrailRequest(BaseModel):
|
||||
"""POST body for the intervene-service full-payload guardrail endpoint (/guardrails/litellm/v2)."""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
event_type: Literal["pre_call", "during_call", "post_call"]
|
||||
metadata: ThirdlawGuardrailRequestMetadata
|
||||
request_url: str | None = None
|
||||
request_headers: Mapping[str, str] | None = None
|
||||
request_body: Mapping[str, object] | None = None
|
||||
response_body: Mapping[str, object] | None = None
|
||||
response_headers: Mapping[str, str] | None = None
|
||||
# The buffered stream beside the assembled body: the events a /v1/responses or chat completions
|
||||
# client received, or the raw SSE text a /v1/messages client received. The service may fold
|
||||
# either side and fall back to the other, without a LiteLLM release in between.
|
||||
response_chunks: Sequence[Mapping[str, object]] | None = None
|
||||
response_sse: str | None = None
|
||||
# Text a /v1/responses stream delivered in delta events that its terminal body does not repeat
|
||||
# (reasoning summaries, tool-call arguments on some providers). Present only when non-empty.
|
||||
streamed_deltas_not_in_body: Sequence[str] | None = None
|
||||
additional_provider_specific_params: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class ThirdlawGuardrailResponse(BaseModel):
|
||||
"""Decision returned by the intervene-service.
|
||||
|
||||
``request_body`` / ``response_body`` are full replacements with shallow top-level
|
||||
overlay semantics: a top-level key that is present is authoritative and must be
|
||||
complete (e.g. a replaced ``choices`` list must carry every choice); omitted
|
||||
top-level keys keep their original values.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
action: Literal["allow", "block", "modify_request", "modify_response"]
|
||||
request_body: Mapping[str, object] | None = None
|
||||
response_body: Mapping[str, object] | None = None
|
||||
response_status: int | None = None
|
||||
message: str = ""
|
||||
|
||||
|
||||
class ThirdlawGuardrailConfigModelOptionalParams(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
additional_headers: str | None = Field(
|
||||
default=None,
|
||||
description="Comma-separated list of inbound request header names whose RAW values ThirdLaw should receive. Every inbound header is always forwarded; credential headers (authorization, x-api-key, cookie) are forwarded as ***REDACTED*** unless listed here. Example: authorization,x-request-id.",
|
||||
)
|
||||
unreachable_fallback: Literal["fail_closed", "fail_open"] = Field(
|
||||
default="fail_closed",
|
||||
description="Controls LiteLLM behavior when ThirdLaw is unreachable (network error, timeout, 502-504). fail_closed blocks the request. fail_open allows the request to continue. A block decision from ThirdLaw always blocks regardless of this setting.",
|
||||
)
|
||||
unscannable_stream_fallback: Literal["fail_closed", "fail_open"] = Field(
|
||||
default="fail_closed",
|
||||
description="Controls LiteLLM behavior when a streamed response cannot be assembled into a scannable shape (for example /v1/responses and text-completion streams). fail_closed refuses the stream. fail_open forwards it unscanned, which lets a caller pick such an endpoint to bypass response moderation.",
|
||||
)
|
||||
send_stream_chunks: bool | None = Field(
|
||||
default=False,
|
||||
description="If true, a finished streamed response is posted both as the assembled provider body in response_body and as the buffered stream beside it: response_chunks for /v1/responses and chat completions, response_sse for /v1/messages. Set to false to post the assembled body alone.",
|
||||
)
|
||||
streaming_buffer_until_moderated: bool | None = Field(
|
||||
default=True,
|
||||
description="If true (default), a streamed response is withheld until ThirdLaw has moderated the assembled response; a modify_response decision is then applied before anything reaches the client.",
|
||||
)
|
||||
streaming_end_of_stream_only: bool | None = Field(
|
||||
default=True,
|
||||
description="If true (default), ThirdLaw is only called once, when a streamed response finishes, instead of periodically during streaming. Set to false to enable interim block-only checks via streaming_sampling_rate.",
|
||||
)
|
||||
streaming_sampling_rate: int | None = Field(
|
||||
default=5,
|
||||
description="When streaming_end_of_stream_only is false, check every Nth streamed chunk (in addition to the final end-of-stream check). Interim checks can only block, not modify. Ignored when streaming_end_of_stream_only is true.",
|
||||
)
|
||||
api_key: str | None = Field(
|
||||
default=None,
|
||||
description="API key for ThirdLaw, sent as a bearer token. Env: THIRDLAW_API_KEY.",
|
||||
)
|
||||
|
||||
|
||||
class ThirdlawGuardrailConfigModel(GuardrailConfigModel[ThirdlawGuardrailConfigModelOptionalParams]):
|
||||
api_base: str | None = Field(
|
||||
default=None,
|
||||
description="ThirdLaw intervene-service base URL; /guardrails/litellm/v2 is appended unless already present. Env: THIRDLAW_API_BASE.",
|
||||
json_schema_extra={
|
||||
"examples": [
|
||||
"https://api.thirdlaw.<your-domain>",
|
||||
]
|
||||
},
|
||||
)
|
||||
guardrail_timeout: int | None = Field(
|
||||
default=60,
|
||||
description="Timeout for the ThirdLaw API request. In seconds.",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def ui_friendly_name() -> str:
|
||||
return "ThirdLaw"
|
||||
|
|
@ -59,8 +59,20 @@ EXCLUDED_TERMINAL_VARS = {
|
|||
"ALACRITTY_SOCKET",
|
||||
}
|
||||
|
||||
# Guardrail integrations whose settings docs live in the separate
|
||||
# BerriAI/litellm-docs repository, so they cannot be added in the same commit
|
||||
# that introduces the integration here. Drop the entry once the docs land.
|
||||
EXCLUDED_PENDING_DOCS_VARS = {
|
||||
"THIRDLAW_API_BASE",
|
||||
"THIRDLAW_API_KEY",
|
||||
}
|
||||
|
||||
EXCLUDED_KEYS = frozenset(
|
||||
EXCLUDED_TERMINAL_VARS | EXCLUDED_GUARD_ONLY_VARS | EXCLUDED_ROLLOUT_FLAGS | EXCLUDED_INTERNAL_TUNING_VARS
|
||||
EXCLUDED_TERMINAL_VARS
|
||||
| EXCLUDED_GUARD_ONLY_VARS
|
||||
| EXCLUDED_ROLLOUT_FLAGS
|
||||
| EXCLUDED_INTERNAL_TUNING_VARS
|
||||
| EXCLUDED_PENDING_DOCS_VARS
|
||||
)
|
||||
|
||||
# Directories to skip (dependencies, venvs, caches) - only scan litellm source
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.anthropic_interface.exceptions import anthropic_error_sse_frame
|
||||
from litellm.caching.caching import DualCache
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.anthropic_endpoints.streaming_model_restamp import (
|
||||
|
|
@ -38,9 +39,7 @@ def _model_from_frame(frame: bytes | str) -> str:
|
|||
|
||||
|
||||
def test_restamps_sse_bytes_frame():
|
||||
restamped = restamp_anthropic_stream_chunk_model(
|
||||
_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1"
|
||||
)
|
||||
restamped = restamp_anthropic_stream_chunk_model(_message_start_frame("claude-haiku-4-5-20251001"), "claude-auto-1")
|
||||
|
||||
assert isinstance(restamped, bytes)
|
||||
assert _model_from_frame(restamped) == "claude-auto-1"
|
||||
|
|
@ -171,6 +170,14 @@ def test_first_non_ping_event_disarms_the_restamper():
|
|||
assert restamper.process(late_message_start) == late_message_start
|
||||
|
||||
|
||||
def test_an_error_frame_passes_through_with_its_status():
|
||||
"""A stream that fails before its first byte is answered with the frame's status, which only
|
||||
the frame object carries; re-encoding it to bytes answered a ThirdLaw 403 block as a 200."""
|
||||
frame = anthropic_error_sse_frame(status_code=403, raw_message="blocked")
|
||||
|
||||
assert AnthropicStreamModelRestamper("claude-auto-1").process(frame) is frame
|
||||
|
||||
|
||||
def test_oversized_unterminated_chunk_flushes_unmodified():
|
||||
blob = b"data: " + b"x" * 70000
|
||||
restamper = AnthropicStreamModelRestamper("claude-auto-1")
|
||||
|
|
|
|||
2709
tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py
Normal file
2709
tests/unit/proxy/guardrails/guardrail_hooks/test_thirdlaw.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,17 +1,225 @@
|
|||
"""Assembly contract for raw Anthropic SSE streams.
|
||||
|
||||
Four guardrails (bedrock, model_armor, tool_permission, thirdlaw) and the OTel request/response
|
||||
recorder all scan the body this module assembles, so anything it drops is invisible to every one
|
||||
of them.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Sequence
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.guardrails.anthropic_sse import (
|
||||
anthropic_sse_chunks_from_body,
|
||||
anthropic_sse_chunks_from_response,
|
||||
assemble_anthropic_sse_body,
|
||||
assemble_anthropic_sse_stream,
|
||||
)
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
|
||||
def _frames(*events: tuple[str, dict[str, object]]) -> list[bytes]:
|
||||
return [f"event: {name}\ndata: {json.dumps(body)}\n\n".encode() for name, body in events]
|
||||
|
||||
|
||||
def _message_start() -> tuple[str, dict[str, object]]:
|
||||
return (
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 0},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_block_start(index: int = 0) -> tuple[str, dict[str, object]]:
|
||||
return (
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": {"type": "tool_use", "id": "toolu_1", "name": "lookup", "input": {}},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _input_delta(partial_json: str, index: int = 0) -> tuple[str, dict[str, object]]:
|
||||
return (
|
||||
"content_block_delta",
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": index,
|
||||
"delta": {"type": "input_json_delta", "partial_json": partial_json},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _assembled_tool_input(*events: tuple[str, dict[str, object]]) -> object:
|
||||
"""The tool block as it lands in the native Anthropic body, which is the shape the /v1/messages
|
||||
scan is handed. The ModelResponse conversion keeps the raw partial JSON by another route, so
|
||||
asserting through that one would pass either way."""
|
||||
body = assemble_anthropic_sse_body(_frames(*events))
|
||||
assert body is not None
|
||||
content = body["content"]
|
||||
assert isinstance(content, Sequence)
|
||||
return content[0]["input"]
|
||||
|
||||
|
||||
def test_complete_tool_arguments_assemble_as_the_object_the_model_sent():
|
||||
arguments = _assembled_tool_input(
|
||||
_message_start(),
|
||||
_tool_block_start(),
|
||||
_input_delta('{"query": "weather'),
|
||||
_input_delta(' in sf"}'),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
assert arguments == {"query": "weather in sf"}
|
||||
|
||||
|
||||
def test_truncated_tool_arguments_keep_the_text_instead_of_dropping_it():
|
||||
"""max_tokens can cut a tool argument mid-JSON. That fragment still reached the client, so
|
||||
discarding it here would hand a caller a way to smuggle text past every scanner above."""
|
||||
arguments = _assembled_tool_input(
|
||||
_message_start(),
|
||||
_tool_block_start(),
|
||||
_input_delta('{"query": "sk-live-TRUNC'),
|
||||
(
|
||||
"message_delta",
|
||||
{"type": "message_delta", "delta": {"stop_reason": "max_tokens"}, "usage": {"output_tokens": 5}},
|
||||
),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
assert "sk-live-TRUNC" in json.dumps(arguments)
|
||||
|
||||
|
||||
def test_a_tool_block_with_no_arguments_assembles_as_an_empty_object():
|
||||
"""A tool called with no arguments is not truncation, so it must not pick up a raw fragment."""
|
||||
arguments = _assembled_tool_input(
|
||||
_message_start(),
|
||||
_tool_block_start(),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
)
|
||||
assert arguments == {}
|
||||
|
||||
|
||||
def _text_block_start(index: int = 0) -> tuple[str, dict[str, object]]:
|
||||
return (
|
||||
"content_block_start",
|
||||
{"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}},
|
||||
)
|
||||
|
||||
|
||||
def _block_delta(delta: dict[str, object], index: int = 0) -> tuple[str, dict[str, object]]:
|
||||
return ("content_block_delta", {"type": "content_block_delta", "index": index, "delta": delta})
|
||||
|
||||
|
||||
def _assembled_content(*events: tuple[str, dict[str, object]]) -> Sequence[dict[str, object]]:
|
||||
body = assemble_anthropic_sse_body(_frames(*events))
|
||||
assert body is not None
|
||||
content = body["content"]
|
||||
assert isinstance(content, Sequence)
|
||||
return content
|
||||
|
||||
|
||||
def test_citation_deltas_land_on_the_text_block_they_annotate():
|
||||
citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0}
|
||||
content = _assembled_content(
|
||||
_message_start(),
|
||||
_text_block_start(),
|
||||
_block_delta({"type": "text_delta", "text": "Blue."}),
|
||||
_block_delta({"type": "citations_delta", "citation": citation}),
|
||||
_block_delta({"type": "citations_delta"}),
|
||||
_block_delta({"type": "some_future_delta", "text": "ignored"}),
|
||||
)
|
||||
assert list(content) == [{"type": "text", "text": "Blue.", "citations": (citation,)}]
|
||||
|
||||
|
||||
def test_a_block_type_with_no_deltas_keeps_its_start_payload():
|
||||
redacted = {"type": "redacted_thinking", "data": "opaque-blob"}
|
||||
content = _assembled_content(
|
||||
_message_start(),
|
||||
("content_block_start", {"type": "content_block_start", "index": 0, "content_block": redacted}),
|
||||
)
|
||||
assert list(content) == [redacted]
|
||||
|
||||
|
||||
def test_malformed_block_events_are_skipped_without_corrupting_the_rest():
|
||||
content = _assembled_content(
|
||||
_message_start(),
|
||||
("content_block_start", {"type": "content_block_start", "content_block": {"type": "text", "text": ""}}),
|
||||
_text_block_start(),
|
||||
("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": "not-a-mapping"}),
|
||||
_block_delta({"type": "text_delta", "text": "kept"}),
|
||||
)
|
||||
assert list(content) == [{"type": "text", "text": "kept"}]
|
||||
|
||||
|
||||
def test_an_undecodable_stream_assembles_to_nothing():
|
||||
"""Nothing scannable comes out of bytes that are not UTF-8, so callers must treat the stream as
|
||||
unassembled rather than scanning a partial body."""
|
||||
frames = _frames(_message_start(), _text_block_start())
|
||||
assert assemble_anthropic_sse_body([*frames[:1], b"\xff\xfe\xfd", *frames[1:]]) is None
|
||||
|
||||
|
||||
def test_a_replayed_body_keeps_its_citations_and_redacted_thinking():
|
||||
"""A guardrail rewrite is replayed to the client from the body, so anything the replay drops is
|
||||
lost to the caller even though the scan saw it: citations lose their sources, and a
|
||||
redacted_thinking block without its data cannot be sent back on the next turn."""
|
||||
citation = {"type": "char_location", "cited_text": "the sky is blue", "document_index": 0}
|
||||
content = [
|
||||
{"type": "redacted_thinking", "data": "opaque-blob"},
|
||||
{"type": "text", "text": "The sky is blue.", "citations": [citation]},
|
||||
{"type": "text", "text": "No source here."},
|
||||
]
|
||||
body: dict[str, object] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": content,
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 7},
|
||||
}
|
||||
replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body)))
|
||||
assert replayed is not None
|
||||
assert json.loads(json.dumps(replayed["content"])) == content
|
||||
|
||||
|
||||
def test_a_tool_input_sent_whole_in_its_block_start_survives_assembly():
|
||||
"""Server tool blocks reach the stream with their full input in ``content_block_start`` and no
|
||||
argument deltas. Blanking that input would replay a query to the client that was never scanned."""
|
||||
body: dict[str, object] = {
|
||||
"id": "msg_1",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5",
|
||||
"content": [
|
||||
{"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "restricted"}}
|
||||
],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 3, "output_tokens": 7},
|
||||
}
|
||||
replayed = assemble_anthropic_sse_body(list(anthropic_sse_chunks_from_body(body)))
|
||||
assert replayed is not None
|
||||
assert json.loads(json.dumps(replayed["content"])) == body["content"]
|
||||
|
||||
|
||||
def _frame(event: str, data: dict[str, object]) -> bytes:
|
||||
return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode()
|
||||
|
||||
|
|
|
|||
122
tests/unit/proxy/guardrails/test_stream_surface.py
Normal file
122
tests/unit/proxy/guardrails/test_stream_surface.py
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.guardrails.stream_surface import (
|
||||
StreamSurface,
|
||||
classify_stream,
|
||||
is_terminal_error_stream,
|
||||
)
|
||||
from litellm.types.llms.openai import (
|
||||
ErrorEvent,
|
||||
ErrorEventError,
|
||||
OutputTextDeltaEvent,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
|
||||
def _chat_chunks() -> tuple[ModelResponseStream, ...]:
|
||||
return (
|
||||
ModelResponseStream(
|
||||
model="gpt-4o-mini",
|
||||
choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="hi"))],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_sse_frames() -> tuple[bytes, ...]:
|
||||
message_start = {
|
||||
"type": "message_start",
|
||||
"message": {"id": "msg_1", "model": "claude-haiku-4-5", "content": [], "usage": {"input_tokens": 3}},
|
||||
}
|
||||
delta = {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hi"}}
|
||||
return (
|
||||
f"event: message_start\ndata: {json.dumps(message_start)}\n\n".encode(),
|
||||
f"event: content_block_delta\ndata: {json.dumps(delta)}\n\n".encode(),
|
||||
)
|
||||
|
||||
|
||||
def _google_sse_frames() -> tuple[bytes, ...]:
|
||||
"""The :streamGenerateContent route marks its stream raw too, but is not Anthropic."""
|
||||
payload = {"candidates": [{"content": {"parts": [{"text": "hi"}], "role": "model"}}]}
|
||||
return (f"data: {json.dumps(payload)}\n\n".encode(),)
|
||||
|
||||
|
||||
def _responses_events() -> tuple[object, ...]:
|
||||
return (
|
||||
OutputTextDeltaEvent(
|
||||
type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA,
|
||||
item_id="msg_1",
|
||||
output_index=0,
|
||||
content_index=0,
|
||||
delta="hi",
|
||||
),
|
||||
ResponseCompletedEvent(
|
||||
type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
|
||||
response=ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=1,
|
||||
model="claude-haiku-4-5",
|
||||
object="response",
|
||||
output=[],
|
||||
parallel_tool_calls=True,
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
top_p=1.0,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"chunks, expected",
|
||||
[
|
||||
(_chat_chunks(), StreamSurface.CHAT_COMPLETIONS),
|
||||
(_anthropic_sse_frames(), StreamSurface.ANTHROPIC_MESSAGES),
|
||||
(_responses_events(), StreamSurface.RESPONSES),
|
||||
(_google_sse_frames(), StreamSurface.OPAQUE_SSE),
|
||||
((), StreamSurface.CHAT_COMPLETIONS),
|
||||
],
|
||||
)
|
||||
def test_classify_stream_names_each_wire_format(chunks, expected):
|
||||
assert classify_stream(chunks) is expected
|
||||
|
||||
|
||||
def test_opaque_sse_is_not_mistaken_for_anthropic():
|
||||
"""Reading a Google stream as Anthropic would refuse it in a format its client cannot parse."""
|
||||
assert classify_stream(_google_sse_frames()) is not StreamSurface.ANTHROPIC_MESSAGES
|
||||
|
||||
|
||||
def test_responses_events_are_classified_from_a_plain_dict_too():
|
||||
"""Some producers hand the hook already-serialized events rather than the pydantic models."""
|
||||
assert classify_stream(({"type": "response.output_text.delta", "delta": "hi"},)) is StreamSurface.RESPONSES
|
||||
|
||||
|
||||
def test_a_responses_error_only_stream_is_a_terminal_error_stream():
|
||||
events = (
|
||||
ErrorEvent(
|
||||
type=ResponsesAPIStreamEvents.ERROR,
|
||||
sequence_number=1,
|
||||
error=ErrorEventError(type="guardrail_error", code="400", message="blocked upstream"),
|
||||
),
|
||||
)
|
||||
assert is_terminal_error_stream(events) is True
|
||||
|
||||
|
||||
def test_an_anthropic_error_only_stream_is_a_terminal_error_stream():
|
||||
frames = (b'event: error\ndata: {"type": "error", "error": {"message": "blocked upstream"}}\n\n',)
|
||||
assert is_terminal_error_stream(frames) is True
|
||||
|
||||
|
||||
def test_a_stream_carrying_content_is_not_a_terminal_error_stream():
|
||||
assert is_terminal_error_stream(_responses_events()) is False
|
||||
assert is_terminal_error_stream(_chat_chunks()) is False
|
||||
assert is_terminal_error_stream(_anthropic_sse_frames()) is False
|
||||
|
||||
|
||||
def test_an_empty_stream_is_not_a_terminal_error_stream():
|
||||
"""all() over nothing is True, so an empty stream would otherwise read as somebody's refusal."""
|
||||
assert is_terminal_error_stream(()) is False
|
||||
84
ui/litellm-dashboard/public/assets/logos/thirdlaw.svg
Normal file
84
ui/litellm-dashboard/public/assets/logos/thirdlaw.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 9.5 KiB |
|
|
@ -322,6 +322,12 @@ export const GUARDRAIL_PRESETS: Record<string, GuardrailPreset> = {
|
|||
mode: "pre_call",
|
||||
defaultOn: false,
|
||||
},
|
||||
thirdlaw: {
|
||||
provider: "ThirdLaw",
|
||||
guardrailNameSuggestion: "ThirdLaw Guardrail",
|
||||
mode: "pre_call",
|
||||
defaultOn: false,
|
||||
},
|
||||
promptguard: {
|
||||
provider: "Promptguard",
|
||||
guardrailNameSuggestion: "PromptGuard",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ const EXPECTED_PARTNER_LOGO_FILES: Record<string, string> = {
|
|||
deepkeep: "deepkeep.svg",
|
||||
repelloai: "repelloai.png",
|
||||
straiker: "straiker.svg",
|
||||
thirdlaw: "thirdlaw.svg",
|
||||
alice: "alice.svg",
|
||||
agent_365: "microsoft_azure.svg",
|
||||
llm_shield_proxy: "llm_shield_proxy.svg",
|
||||
|
|
|
|||
|
|
@ -507,6 +507,16 @@ export const PARTNER_GUARDRAIL_CARDS: GuardrailCardInfo[] = [
|
|||
tags: ["Security", "Policy", "Prompt Injection"],
|
||||
providerKey: "Repelloai",
|
||||
},
|
||||
{
|
||||
id: "thirdlaw",
|
||||
name: "ThirdLaw",
|
||||
description:
|
||||
"Runtime policy enforcement for LiteLLM traffic. Evaluates requests and responses against centrally managed policies called Laws, then returns a decision to continue, modify, or block.",
|
||||
category: "partner",
|
||||
logo: guardrailLogoMap["ThirdLaw"],
|
||||
tags: ["Policy Enforcement", "Runtime Security", "Intervention"],
|
||||
providerKey: "ThirdLaw",
|
||||
},
|
||||
{
|
||||
id: "straiker",
|
||||
name: "Straiker",
|
||||
|
|
|
|||
|
|
@ -224,6 +224,20 @@ describe("guardrail_info_helpers", () => {
|
|||
expect(result.displayName).toBe("RepelloAI Argus");
|
||||
expect(result.logo).toContain("repelloai.png");
|
||||
});
|
||||
|
||||
it("should return thirdlaw logo when ui_friendly_name is ThirdLaw", () => {
|
||||
populateGuardrailProviders({
|
||||
thirdlaw: { ui_friendly_name: "ThirdLaw" },
|
||||
});
|
||||
populateGuardrailProviderMap({
|
||||
thirdlaw: { ui_friendly_name: "ThirdLaw" },
|
||||
});
|
||||
|
||||
const result = getGuardrailLogoAndName("thirdlaw");
|
||||
|
||||
expect(result.displayName).toBe("ThirdLaw");
|
||||
expect(result.logo).toContain("thirdlaw.svg");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatGuardrailMode", () => {
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import repelloAiLogo from "../../../../../public/assets/logos/repelloai.png";
|
|||
import straikerLogo from "../../../../../public/assets/logos/straiker.svg";
|
||||
import wingbackLogo from "../../../../../public/assets/logos/wingback.png";
|
||||
import xecguardLogo from "../../../../../public/assets/logos/xecguard.svg";
|
||||
import thirdlawLogo from "../../../../../public/assets/logos/thirdlaw.svg";
|
||||
import zscalerLogo from "../../../../../public/assets/logos/zscaler.svg";
|
||||
|
||||
// Legacy enum - keeping for backward compatibility
|
||||
|
|
@ -88,6 +89,7 @@ export const guardrail_provider_map: Record<string, string> = {
|
|||
Deepkeep: "deepkeep",
|
||||
QostodianNexus: "qostodian_nexus",
|
||||
Repelloai: "repelloai",
|
||||
ThirdLaw: "thirdlaw",
|
||||
Alice: "alice",
|
||||
"LLM Shield Proxy": "llm_shield_proxy",
|
||||
Conduct: "conduct",
|
||||
|
|
@ -290,6 +292,7 @@ export const shouldRenderDecisionModelFields = (provider: string | null) => {
|
|||
};
|
||||
|
||||
export const guardrailLogoMap = {
|
||||
ThirdLaw: thirdlawLogo.src,
|
||||
"Zscaler AI Guard": zscalerLogo.src,
|
||||
"Presidio PII": microsoftAzureLogo.src,
|
||||
"Bedrock Guardrail": bedrockLogo.src,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue