diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py b/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py index 9391cf60cc3..e372039936c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/__init__.py @@ -38,6 +38,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail" def initialize_guardrail_v2(litellm_params: "LitellmParams", guardrail: "Guardrail"): import litellm + extras: Final = litellm_params.model_extra or {} _noma_v2_callback: Final = NomaV2Guardrail( guardrail_name=guardrail.get("guardrail_name", ""), api_key=litellm_params.api_key, @@ -49,6 +50,10 @@ def initialize_guardrail_v2(litellm_params: "LitellmParams", guardrail: "Guardra event_hook=litellm_params.mode, default_on=litellm_params.default_on, timeout=litellm_params.timeout, + streaming_buffer_until_moderated=extras.get("streaming_buffer_until_moderated"), + streaming_end_of_stream_only=extras.get("streaming_end_of_stream_only"), + streaming_sampling_rate=extras.get("streaming_sampling_rate"), + streaming_buffer_release_on_scan=extras.get("streaming_buffer_release_on_scan"), ) litellm.logging_callback_manager.add_litellm_callback(_noma_v2_callback) diff --git a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py index 37a33023d54..360752d201a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py +++ b/litellm/proxy/guardrails/guardrail_hooks/noma/noma_v2.py @@ -26,6 +26,7 @@ from litellm.llms.custom_httpx.http_handler import ( from litellm.proxy.guardrails.guardrail_hooks.noma.noma import NomaBlockedMessage from litellm.types.guardrail_base_init import GuardrailBaseInitKwargs from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.proxy.guardrails.guardrail_hooks.noma import NomaV2StreamingParams from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailStatus if TYPE_CHECKING: @@ -63,6 +64,10 @@ class NomaV2Guardrail(CustomGuardrail): monitor_mode: bool | None = None, block_failures: bool | None = None, gateway_name: str | None = None, + streaming_buffer_until_moderated: bool | None = None, + streaming_end_of_stream_only: bool | None = None, + streaming_sampling_rate: int | None = None, + streaming_buffer_release_on_scan: bool | None = None, **kwargs: Any, ) -> None: self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback) @@ -86,6 +91,24 @@ class NomaV2Guardrail(CustomGuardrail): if self._requires_api_key(api_base=self.api_base) and not self.api_key: raise ValueError("Noma v2 guardrail requires api_key when using Noma SaaS endpoint") + streaming: Final = NomaV2StreamingParams.model_validate( + { + "streaming_buffer_until_moderated": streaming_buffer_until_moderated, + "streaming_end_of_stream_only": streaming_end_of_stream_only, + "streaming_sampling_rate": streaming_sampling_rate, + "streaming_buffer_release_on_scan": streaming_buffer_release_on_scan, + }, + context={"guardrail_name": kwargs.get("guardrail_name")}, + ) + if streaming.streaming_buffer_until_moderated is not None: + self.streaming_buffer_until_moderated: bool = streaming.streaming_buffer_until_moderated + if streaming.streaming_end_of_stream_only is not None: + self.streaming_end_of_stream_only: bool = streaming.streaming_end_of_stream_only + if streaming.streaming_sampling_rate is not None: + self.streaming_sampling_rate: int = streaming.streaming_sampling_rate + if streaming.streaming_buffer_release_on_scan is not None: + self.streaming_buffer_release_on_scan: bool = streaming.streaming_buffer_release_on_scan + kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks())) base_kwargs: Final[GuardrailBaseInitKwargs] = kwargs diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py index ef22f73810f..992815bf454 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/noma.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/noma.py @@ -1,4 +1,16 @@ -from pydantic import Field +from typing import Final + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + ValidationInfo, + ValidatorFunctionWrapHandler, + field_validator, +) + +from litellm._logging import verbose_proxy_logger from .base import GuardrailConfigModel @@ -55,3 +67,35 @@ class NomaV2GuardrailConfigModel(GuardrailConfigModel): @staticmethod def ui_friendly_name() -> str: return "Noma Security v2" + + +class NomaV2StreamingParams(BaseModel): + """Streaming settings a Noma v2 guardrail reads from the top level of its litellm_params. + + A key left unset, or set to a value that does not validate, stays None, so the unified + streaming hook keeps its own default for it and the guardrail still loads. + """ + + model_config = ConfigDict(frozen=True) + + streaming_buffer_until_moderated: bool | None = None + streaming_end_of_stream_only: bool | None = None + streaming_sampling_rate: int | None = Field(default=None, ge=1) + streaming_buffer_release_on_scan: bool | None = None + + @field_validator("*", mode="wrap") + @classmethod + def _ignore_invalid_value( + cls, value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo + ) -> object: + try: + validated: Final = handler(value) + except ValidationError: + verbose_proxy_logger.warning( + "Noma v2 guardrail %s: ignoring %s=%.100r, which is not a valid value; the default applies", + info.context.get("guardrail_name") if info.context else None, + info.field_name, + value, + ) + return None + return validated diff --git a/tests/integration/streaming/test_noma_v2_streaming_settings.py b/tests/integration/streaming/test_noma_v2_streaming_settings.py new file mode 100644 index 00000000000..6354a2c9135 --- /dev/null +++ b/tests/integration/streaming/test_noma_v2_streaming_settings.py @@ -0,0 +1,1009 @@ +import asyncio +import json +import threading +import time +import uuid +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass, field +from itertools import chain +from pathlib import Path +from typing import Final, Literal, TypeAlias, cast + +import anthropic +import httpx +import openai +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, list_value, object_value, string_value +from integration._support.process import OwnedProxy, WorkerHealthcheck, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +Endpoint: TypeAlias = Literal["chat", "messages", "responses"] +Spelling: TypeAlias = Literal["noma_v2", "noma_use_v2", "stored", "custom"] +Variant: TypeAlias = Literal[ + "default", + "buffered", + "end_only", + "sparse", + "dense", + "release", + "buffered_word", + "sparse_word", + "invalid_rate_buffered", + "invalid_rate_end_only", + "release_invalid", + "unset", + "off", + "off_words", + "invalid", + "release_alone", + "fail_open", + "pre_call", +] +Action: TypeAlias = Literal["pass", "block", "mask", "late", "outage", "slow", "reject"] +Client: TypeAlias = Literal["sync", "async"] +SdkEvent: TypeAlias = tuple[Literal["text", "failed", "status", "error"], str] + +ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") +SPELLINGS: Final[tuple[Spelling, ...]] = ("noma_v2", "noma_use_v2", "stored", "custom") +MODELS: Final[Mapping[Endpoint, str]] = { + "chat": "noma-chat", + "messages": "noma-messages", + "responses": "noma-responses", +} +PATHS: Final[Mapping[Endpoint, str]] = { + "chat": "/v1/chat/completions", + "messages": "/v1/messages", + "responses": "/v1/responses", +} +SECRET: Final = "jane@example.com" +MASK: Final = "[EMAIL]" +BLOCKED: Final = "Request blocked by Noma guardrail" +SLOW_SECONDS: Final = 1.5 +SAMPLED_SCANS: Final = range(2, 7) +BUFFERED_BLOCK_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 400, "messages": 400, "responses": 200} +BUFFERED_OUTAGE_STATUS: Final[Mapping[Endpoint, int]] = {"chat": 503, "messages": 503, "responses": 200} +PIECES: Final = ( + " Contact", + " me", + " at", + " work", + " via", + f" {SECRET}", + " for", + " the", + " quarterly", + " report", + " and", + " the", + " budget", + ".", +) +STREAMING_KEYS: Final = ( + "streaming_buffer_until_moderated", + "streaming_end_of_stream_only", + "streaming_sampling_rate", + "streaming_buffer_release_on_scan", +) +INVALID: Final[Mapping[str, JsonValue]] = { + "streaming_buffer_until_moderated": "maybe", + "streaming_end_of_stream_only": "maybe", + "streaming_sampling_rate": 0, + "streaming_buffer_release_on_scan": "maybe", +} +SETTINGS: Final[Mapping[Variant, Mapping[str, JsonValue]]] = { + "default": {}, + "buffered": {"streaming_buffer_until_moderated": True}, + "end_only": {"streaming_end_of_stream_only": True}, + "sparse": {"streaming_sampling_rate": 1000}, + "dense": {"streaming_sampling_rate": 1}, + "release": {"streaming_buffer_until_moderated": True, "streaming_buffer_release_on_scan": True}, + "buffered_word": {"streaming_buffer_until_moderated": "true"}, + "sparse_word": {"streaming_sampling_rate": "1000"}, + "invalid_rate_buffered": {"streaming_buffer_until_moderated": True, "streaming_sampling_rate": -5}, + "invalid_rate_end_only": {"streaming_end_of_stream_only": True, "streaming_sampling_rate": "often"}, + "release_invalid": {"streaming_buffer_until_moderated": True, "streaming_buffer_release_on_scan": "maybe"}, + "unset": dict.fromkeys(STREAMING_KEYS), + "off": { + "streaming_buffer_until_moderated": False, + "streaming_end_of_stream_only": False, + "streaming_buffer_release_on_scan": False, + }, + "off_words": { + "streaming_buffer_until_moderated": "false", + "streaming_end_of_stream_only": "false", + "streaming_buffer_release_on_scan": "false", + }, + "invalid": INVALID, + "release_alone": {"streaming_buffer_release_on_scan": True}, + "fail_open": {"streaming_buffer_until_moderated": True, "block_failures": False}, + "pre_call": {"mode": "pre_call", "streaming_buffer_until_moderated": True}, +} +SUBCLASS_MODULE: Final = "noma_v2_subclass" +SUBCLASS_SOURCE: Final = """from litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2 import NomaV2Guardrail + + +class NomaV2Subclass(NomaV2Guardrail): + async def apply_guardrail(self, *args, **kwargs): + return await super().apply_guardrail(*args, **kwargs) +""" +SPELLING_PARAMS: Final[Mapping[Spelling, Mapping[str, JsonValue]]] = { + "noma_v2": {"guardrail": "noma_v2"}, + "noma_use_v2": {"guardrail": "noma", "use_v2": True}, + "stored": {"guardrail": "noma_v2"}, + "custom": {"guardrail": f"{SUBCLASS_MODULE}.NomaV2Subclass"}, +} + + +def _texts(value: JsonValue) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(chain.from_iterable(_texts(item) for item in value)) + if isinstance(value, dict): + return tuple(chain.from_iterable(_texts(item) for item in value.values())) + return () + + +def _marker(body: Mapping[str, JsonValue]) -> str: + return next(text for text in _texts(dict(body)) if text.startswith("noma-")) + + +def _full_text(marker: str) -> str: + return marker + "".join(PIECES) + + +def _action(marker: str) -> Action: + return cast(Action, marker.split("-")[1]) + + +def _scan_action(text: str) -> Action: + marker: Final = next((word for word in text.split(" ") if word.startswith("noma-")), None) + return "pass" if marker is None else _action(marker) + + +def _event(name: str | None, payload: Mapping[str, JsonValue]) -> bytes: + prefix: Final = f"event: {name}\n" if name is not None else "" + return f"{prefix}data: {json.dumps(payload, separators=(',', ':'))}\n\n".encode() + + +def _chat_stream(marker: str, choices: int) -> tuple[bytes, ...]: + chunk: Final = {"id": f"chatcmpl-{marker}", "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final = ( + {"role": "assistant", "content": marker}, + *({"content": piece} for piece in PIECES), + ) + return ( + *( + _event(None, {**chunk, "choices": [{"index": index, "delta": delta, "finish_reason": None}]}) + for delta in deltas + for index in range(choices) + ), + _event( + None, + {**chunk, "choices": [{"index": index, "delta": {}, "finish_reason": "stop"} for index in range(choices)]}, + ), + b"data: [DONE]\n\n", + ) + + +def _message(marker: str) -> dict[str, JsonValue]: + return { + "id": f"msg-{marker}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": _full_text(marker)}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 15}, + } + + +def _messages_stream(marker: str) -> tuple[bytes, ...]: + events: Final[tuple[dict[str, JsonValue], ...]] = ( + { + "type": "message_start", + "message": { + **_message(marker), + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 2, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + *( + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}} + for text in (marker, *PIECES) + ), + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 15}}, + {"type": "message_stop"}, + ) + return tuple(_event(string_value(event["type"]), event) for event in events) + + +def _response_output(marker: str) -> dict[str, JsonValue]: + return { + "type": "message", + "id": f"msg-{marker}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _full_text(marker), "annotations": []}], + } + + +def _response(marker: str) -> dict[str, JsonValue]: + return { + "id": f"resp-{marker}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [_response_output(marker)], + "usage": {"input_tokens": 2, "output_tokens": 15, "total_tokens": 17}, + } + + +def _responses_stream(marker: str) -> tuple[bytes, ...]: + item_id: Final = f"msg-{marker}" + text: Final = _full_text(marker) + part: Final[dict[str, JsonValue]] = {"type": "output_text", "text": text, "annotations": []} + output: Final = _response_output(marker) + response: Final = _response(marker) + position: Final[dict[str, JsonValue]] = {"item_id": item_id, "output_index": 0, "content_index": 0} + events: Final[tuple[dict[str, JsonValue], ...]] = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + {"type": "response.in_progress", "response": {**response, "status": "in_progress", "output": []}}, + { + "type": "response.output_item.added", + "item": {**output, "status": "in_progress", "content": []}, + "output_index": 0, + }, + {"type": "response.content_part.added", **position, "part": {**part, "text": ""}}, + *({"type": "response.output_text.delta", **position, "delta": delta} for delta in (marker, *PIECES)), + {"type": "response.output_text.done", **position, "text": text}, + {"type": "response.content_part.done", **position, "part": part}, + {"type": "response.output_item.done", "item": output, "output_index": 0}, + {"type": "response.completed", "response": response}, + ) + return tuple( + _event(string_value(event["type"]), {**event, "sequence_number": index}) for index, event in enumerate(events) + ) + + +def _completed(target: str, marker: str) -> dict[str, JsonValue]: + if target == PATHS["messages"]: + return _message(marker) + if target == PATHS["responses"]: + return _response(marker) + return { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": _full_text(marker)}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 15, "total_tokens": 17}, + } + + +def _rejection(target: str) -> dict[str, JsonValue]: + message: Final = "synthetic upstream rejection" + if target == PATHS["messages"]: + return {"type": "error", "error": {"type": "invalid_request_error", "message": message}} + return {"error": {"message": message, "type": "invalid_request_error", "param": None, "code": None}} + + +def _provider(request: Request) -> Reply: + body: Final = JSON_OBJECT.validate_json(request.body) + marker: Final = _marker(body) + if request.target not in PATHS.values(): + return Reply(status=404, body=json.dumps({"error": f"unexpected target {request.target}"}).encode()) + if _action(marker) == "reject": + return Reply(status=400, body=json.dumps(_rejection(request.target)).encode()) + if body.get("stream") is not True: + return Reply(body=json.dumps(_completed(request.target, marker)).encode()) + choices: Final = body.get("n", 1) + assert isinstance(choices, int), request.body + streams: Final = { + PATHS["chat"]: _chat_stream(marker, choices), + PATHS["messages"]: _messages_stream(marker), + PATHS["responses"]: _responses_stream(marker), + } + return Reply(content_type="text/event-stream", chunks=streams[request.target], pause_between_chunks=0.05) + + +@dataclass(frozen=True, slots=True) +class Scan: + guardrail: str + input_type: str + text: str + + +def _verdict(action: Action, text: str, earlier: int, texts: Sequence[str]) -> dict[str, JsonValue]: + if action == "block" and SECRET in text: + return {"action": "BLOCKED", "reason": "synthetic email policy"} + if action == "late" and earlier > 0: + return {"action": "BLOCKED", "reason": "synthetic late verdict"} + if action == "mask" and SECRET in text: + return {"action": "GUARDRAIL_INTERVENED", "texts": [part.replace(SECRET, MASK) for part in texts]} + return {"action": "NONE"} + + +@dataclass(slots=True) +class NomaDouble: + """Scripted Noma v2 `/litellm/guardrail` endpoint; the action word in the stream's marker picks the verdict.""" + + scans: list[Scan] = field(default_factory=list) + lock: threading.Lock = field(default_factory=threading.Lock) + down: threading.Event = field(default_factory=threading.Event) + + def respond(self, request: Request) -> Reply: + guardrail: Final = request.target.strip("/").split("/")[0] + assert request.target == f"/{guardrail}/litellm/guardrail", request.target + body: Final = JSON_OBJECT.validate_json(request.body) + texts: Final = tuple(string_value(text) for text in list_value(object_value(body["inputs"]).get("texts", []))) + text: Final = "".join(texts) + scan: Final = Scan(guardrail, string_value(body["input_type"]), text) + with self.lock: + earlier: Final = sum(1 for seen in self.scans if seen.text.split(" ")[0] == text.split(" ")[0]) + self.scans.append(scan) + action: Final = _scan_action(text) + if self.down.is_set() or action == "outage": + return Reply(status=503, body=json.dumps({"detail": "synthetic Noma outage"}).encode()) + if action == "slow": + time.sleep(SLOW_SECONDS) + return Reply(body=json.dumps(_verdict(action, text, earlier, texts)).encode()) + + def for_marker(self, marker: str) -> tuple[Scan, ...]: + with self.lock: + return tuple(scan for scan in self.scans if scan.text.startswith(marker)) + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: OwnedProxy + noma: NomaDouble + noma_url: str + provider: Wire + names: Mapping[tuple[Spelling, Variant], str] + stored: Mapping[Variant, str] + + @property + def gateway(self) -> Gateway: + return self.proxy.gateway + + +def _guardrail_params(noma_url: str, name: str, spelling: Spelling, variant: Variant) -> dict[str, JsonValue]: + return { + **SPELLING_PARAMS[spelling], + "mode": "post_call", + "default_on": False, + "api_base": f"{noma_url}/{name}", + "api_key": "synthetic-noma-key", + **SETTINGS[variant], + } + + +def _config(provider_url: str, noma_url: str, names: Mapping[tuple[Spelling, Variant], str]) -> dict[str, JsonValue]: + base: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + return { + **base, + "guardrails": [ + {"guardrail_name": name, "litellm_params": _guardrail_params(noma_url, name, spelling, variant)} + for (spelling, variant), name in names.items() + if spelling != "stored" + ], + "model_list": [ + { + "model_name": MODELS["chat"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider_url}/v1", + "api_key": "synthetic-provider-key", + }, + }, + { + "model_name": MODELS["messages"], + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5-20250929", + "api_base": provider_url, + "api_key": "synthetic-provider-key", + }, + }, + { + "model_name": MODELS["responses"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider_url}/v1", + "api_key": "synthetic-provider-key", + }, + }, + ], + } + + +def _store(gateway: Gateway, noma_url: str, name: str, variant: Variant) -> str: + response: Final = gateway.request( + "POST", + "/guardrails", + {"guardrail": {"guardrail_name": name, "litellm_params": _guardrail_params(noma_url, name, "stored", variant)}}, + ) + assert response.status_code == 200, response.text + return string_value(JSON_OBJECT.validate_json(response.content)["guardrail_id"]) + + +@pytest.fixture(scope="module") +def rig(session_gateway: Gateway, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]: + noma: Final = NomaDouble() + run: Final = uuid.uuid4().hex[:8] + names: Final = { + (spelling, variant): f"{spelling}-{variant}-{run}".replace("_", "-") + for spelling in SPELLINGS + for variant in SETTINGS + } + with wire_server(_provider) as provider, wire_server(noma.respond) as noma_wire: + stored: Final = { + variant: _store(session_gateway, noma_wire.url, name, variant) + for (spelling, variant), name in names.items() + if spelling == "stored" + } + try: + directory: Final = tmp_path_factory.mktemp("noma-v2-streaming") + (directory / f"{SUBCLASS_MODULE}.py").write_text(SUBCLASS_SOURCE) + config_path: Final = directory / "noma-v2-streaming.yaml" + config_path.write_text(yaml.safe_dump(_config(provider.url, noma_wire.url, names))) + with owned_proxy_process( + session_gateway, + directory, + {}, + config=config_path, + workers=2, + healthcheck=WorkerHealthcheck.LENIENT, + ) as proxy: + yield Rig(proxy, noma, noma_wire.url, provider, names, stored) + finally: + for identity in stored.values(): + deleted = session_gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +@dataclass(frozen=True, slots=True) +class Streamed: + marker: str + response: httpx.Response + content: str + scans: tuple[Scan, ...] + + +def _event_text(event: Mapping[str, JsonValue], endpoint: Endpoint) -> str: + if endpoint == "responses": + return string_value(event.get("delta", "")) if event.get("type") == "response.output_text.delta" else "" + if endpoint == "messages": + return string_value(object_value(event.get("delta", {})).get("text", "")) + return "".join( + string_value(object_value(object_value(choice).get("delta", {})).get("content", "") or "") + for choice in list_value(event.get("choices", [])) + ) + + +def _line_text(line: str, endpoint: Endpoint) -> str: + return ( + _event_text(JSON_OBJECT.validate_json(line.removeprefix("data: ")), endpoint) + if line.startswith("data: {") + else "" + ) + + +def _body(endpoint: Endpoint, marker: str, guardrail: str, *, stream: bool = True) -> dict[str, JsonValue]: + prompt: Final[dict[str, JsonValue]] = ( + {"input": marker} if endpoint == "responses" else {"messages": [{"role": "user", "content": marker}]} + ) + return { + "model": MODELS[endpoint], + **prompt, + **({"max_tokens": 64} if endpoint == "messages" else {}), + "stream": stream, + "guardrails": [guardrail], + } + + +def _new_marker(action: Action) -> str: + return f"noma-{action}-{uuid.uuid4().hex}" + + +def _send( + rig: Rig, + endpoint: Endpoint, + guardrail: str, + action: Action, + *, + input_type: str = "response", + extra: Mapping[str, JsonValue] | None = None, + via: Gateway | None = None, +) -> Streamed: + marker: Final = _new_marker(action) + response: Final = (via or rig.gateway).request( + "POST", PATHS[endpoint], {**_body(endpoint, marker, guardrail), **(extra or {})} + ) + scans: Final = rig.noma.for_marker(marker) + assert all(scan.guardrail == guardrail and scan.input_type == input_type for scan in scans), scans + content: Final = "".join(_line_text(line, endpoint) for line in response.text.splitlines()) + return Streamed(marker, response, content, scans) + + +def _stream( + rig: Rig, + endpoint: Endpoint, + spelling: Spelling, + variant: Variant, + action: Action, + status: int = 200, + *, + input_type: str = "response", + extra: Mapping[str, JsonValue] | None = None, +) -> Streamed: + streamed: Final = _send(rig, endpoint, rig.names[(spelling, variant)], action, input_type=input_type, extra=extra) + assert streamed.response.status_code == status, streamed.response.text + return streamed + + +def _burst(rig: Rig, cases: Sequence[tuple[Endpoint, Action]], guardrail: str) -> tuple[Streamed, ...]: + with ThreadPoolExecutor(max_workers=len(cases)) as pool: + return tuple(pool.map(lambda case: _send(rig, case[0], guardrail, case[1]), cases)) + + +def _attempt(rig: Rig, endpoint: Endpoint, guardrail: str, action: Action) -> str: + try: + return _send(rig, endpoint, guardrail, action).response.text + except httpx.TransportError as error: + return f"transport error: {type(error).__name__}" + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +@pytest.mark.parametrize("variant", ["buffered", "buffered_word", "invalid_rate_buffered", "release_invalid"]) +def test_buffer_until_moderated_keeps_flagged_text_from_the_client( + rig: Rig, endpoint: Endpoint, spelling: Spelling, variant: Variant +) -> None: + unbuffered: Final = _stream(rig, endpoint, spelling, "default", "block") + assert SECRET in unbuffered.content, unbuffered.response.text + assert BLOCKED in unbuffered.response.text, unbuffered.response.text + + buffered: Final = _stream(rig, endpoint, spelling, variant, "block", BUFFERED_BLOCK_STATUS[endpoint]) + assert buffered.content == "", buffered.response.text + assert SECRET not in buffered.response.text, buffered.response.text + assert BLOCKED in buffered.response.text, buffered.response.text + assert [scan.text for scan in buffered.scans] == [_full_text(buffered.marker)], buffered.scans + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +@pytest.mark.parametrize("variant", ["end_only", "sparse", "sparse_word", "invalid_rate_end_only"]) +def test_one_scan_settings_scan_the_finished_stream_once( + rig: Rig, endpoint: Endpoint, spelling: Spelling, variant: Variant +) -> None: + sampled: Final = _stream(rig, endpoint, spelling, "default", "pass") + assert sampled.content == _full_text(sampled.marker), sampled.response.text + assert len(sampled.scans) > 1, sampled.scans + + configured: Final = _stream(rig, endpoint, spelling, variant, "pass") + assert configured.content == _full_text(configured.marker), configured.response.text + assert [scan.text for scan in configured.scans] == [_full_text(configured.marker)], configured.scans + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_sampling_rate_one_scans_every_new_piece(rig: Rig, endpoint: Endpoint, spelling: Spelling) -> None: + sampled: Final = _stream(rig, endpoint, spelling, "default", "pass") + assert len(sampled.scans) < len(PIECES), sampled.scans + + dense: Final = _stream(rig, endpoint, spelling, "dense", "pass") + assert dense.content == _full_text(dense.marker), dense.response.text + scanned: Final = {scan.text for scan in dense.scans} + prefixes: Final = {dense.marker + "".join(PIECES[:count]) for count in range(len(PIECES) + 1)} + assert scanned <= prefixes, sorted(scanned - prefixes) + if endpoint == "messages": + # /v1/messages hands the hook raw network reads, so two deltas read together are one chunk + assert len(scanned) > max(SAMPLED_SCANS), sorted(scanned) + else: + assert prefixes <= scanned, sorted(prefixes - scanned) + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_release_on_scan_releases_only_scanned_windows(rig: Rig, endpoint: Endpoint, spelling: Spelling) -> None: + released: Final = _stream(rig, endpoint, spelling, "release", "late") + assert len(released.scans) == 2, released.scans + assert released.content == released.scans[0].text, (released.response.text, released.scans) + assert released.content and _full_text(released.marker).startswith(released.content), released.response.text + assert BLOCKED in released.response.text, released.response.text + + held: Final = _stream(rig, endpoint, spelling, "buffered", "late") + assert held.content == _full_text(held.marker), held.response.text + assert len(held.scans) == 1, held.scans + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_invalid_release_setting_keeps_the_buffered_stream_held( + rig: Rig, endpoint: Endpoint, spelling: Spelling +) -> None: + held: Final = _stream(rig, endpoint, spelling, "release_invalid", "late") + assert held.content == _full_text(held.marker), held.response.text + assert len(held.scans) == 1, held.scans + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +@pytest.mark.parametrize("variant", ["unset", "off", "off_words", "invalid", "release_alone"]) +def test_unset_off_and_invalid_settings_keep_the_sampled_live_stream( + rig: Rig, endpoint: Endpoint, spelling: Spelling, variant: Variant +) -> None: + configured: Final = _stream(rig, endpoint, spelling, variant, "pass") + assert configured.content == _full_text(configured.marker), configured.response.text + assert len(configured.scans) in SAMPLED_SCANS, configured.scans + assert configured.scans[-1].text == _full_text(configured.marker), configured.scans + + leaked: Final = _stream(rig, endpoint, spelling, variant, "block") + assert SECRET in leaked.content, leaked.response.text + assert BLOCKED in leaked.response.text, leaked.response.text + + +@pytest.mark.parametrize("spelling", SPELLINGS) +def test_invalid_settings_are_logged_with_the_guardrail_name(rig: Rig, spelling: Spelling) -> None: + name: Final = rig.names[(spelling, "invalid")] + log: Final = rig.proxy.log.read_text() + expected: Final = tuple(f"Noma v2 guardrail {name}: ignoring {key}={value!r}" for key, value in INVALID.items()) + assert [line for line in expected if line not in log] == [], name + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_noma_outage_fails_closed_before_any_text_when_buffered( + rig: Rig, endpoint: Endpoint, spelling: Spelling +) -> None: + closed: Final = _stream(rig, endpoint, spelling, "buffered", "outage", BUFFERED_OUTAGE_STATUS[endpoint]) + assert closed.content == "", closed.response.text + assert endpoint != "responses" or "response.failed" in closed.response.text, closed.response.text + assert [scan.text for scan in closed.scans] == [_full_text(closed.marker)], closed.scans + + opened: Final = _stream(rig, endpoint, spelling, "fail_open", "outage") + assert opened.content == _full_text(opened.marker), opened.response.text + assert [scan.text for scan in opened.scans] == [_full_text(opened.marker)], opened.scans + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_upstream_rejection_reaches_the_client_without_a_scan(rig: Rig, endpoint: Endpoint) -> None: + rejected: Final = _stream(rig, endpoint, "noma_v2", "buffered", "reject", 400) + assert "synthetic upstream rejection" in rejected.response.text, rejected.response.text + assert rejected.scans == (), rejected.scans + + +@pytest.mark.parametrize("spelling", SPELLINGS) +def test_buffered_block_withholds_every_choice(rig: Rig, spelling: Spelling) -> None: + unbuffered: Final = _stream(rig, "chat", spelling, "default", "block", extra={"n": 2}) + assert SECRET in unbuffered.content, unbuffered.response.text + + buffered: Final = _stream(rig, "chat", spelling, "buffered", "block", 400, extra={"n": 2}) + assert buffered.content == "", buffered.response.text + assert SECRET not in buffered.response.text, buffered.response.text + assert BLOCKED in buffered.response.text, buffered.response.text + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_pre_call_guardrail_scans_the_request_and_streams_live( + rig: Rig, endpoint: Endpoint, spelling: Spelling +) -> None: + streamed: Final = _stream(rig, endpoint, spelling, "pre_call", "pass", input_type="request") + assert streamed.content == _full_text(streamed.marker), streamed.response.text + assert [scan.text for scan in streamed.scans] == [streamed.marker], streamed.scans + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +@pytest.mark.parametrize("variant", ["default", "buffered", "end_only"]) +def test_non_streaming_post_call_ignores_the_streaming_settings(rig: Rig, endpoint: Endpoint, variant: Variant) -> None: + guardrail: Final = rig.names[("noma_v2", variant)] + passed: Final = _send(rig, endpoint, guardrail, "pass", extra={"stream": False}) + assert passed.response.status_code == 200, passed.response.text + assert _full_text(passed.marker) in passed.response.text, passed.response.text + assert [scan.text for scan in passed.scans] == [_full_text(passed.marker)], passed.scans + + blocked: Final = _send(rig, endpoint, guardrail, "block", extra={"stream": False}) + assert blocked.response.status_code == 400, blocked.response.text + assert SECRET not in blocked.response.text, blocked.response.text + assert BLOCKED in blocked.response.text, blocked.response.text + + +@dataclass(frozen=True, slots=True) +class SdkOutcome: + text: str + status: int | None + failed: bool + + +def _outcome(events: Sequence[SdkEvent]) -> SdkOutcome: + return SdkOutcome( + "".join(text for kind, text in events if kind == "text"), + next((int(text) for kind, text in events if kind == "status"), None), + any(kind in ("failed", "error") for kind, _ in events), + ) + + +def _openai_events(client: openai.OpenAI, endpoint: Endpoint, marker: str, guardrail: str) -> Iterator[SdkEvent]: + extra: Final[dict[str, object]] = {"guardrails": [guardrail]} + if endpoint == "chat": + for chunk in client.chat.completions.create( + model=MODELS["chat"], messages=[{"role": "user", "content": marker}], stream=True, extra_body=extra + ): + yield from (("text", choice.delta.content or "") for choice in chunk.choices) + return + for event in client.responses.create(model=MODELS["responses"], input=marker, stream=True, extra_body=extra): + if event.type == "response.output_text.delta": + yield "text", event.delta + if event.type == "response.failed": + yield "failed", event.type + + +async def _async_openai_events( + client: openai.AsyncOpenAI, endpoint: Endpoint, marker: str, guardrail: str +) -> AsyncIterator[SdkEvent]: + extra: Final[dict[str, object]] = {"guardrails": [guardrail]} + if endpoint == "chat": + async for chunk in await client.chat.completions.create( + model=MODELS["chat"], messages=[{"role": "user", "content": marker}], stream=True, extra_body=extra + ): + for choice in chunk.choices: + yield "text", choice.delta.content or "" + return + async for event in await client.responses.create( + model=MODELS["responses"], input=marker, stream=True, extra_body=extra + ): + if event.type == "response.output_text.delta": + yield "text", event.delta + if event.type == "response.failed": + yield "failed", event.type + + +def _anthropic_events(client: anthropic.Anthropic, marker: str, guardrail: str) -> Iterator[SdkEvent]: + for event in client.messages.create( + model=MODELS["messages"], + max_tokens=64, + messages=[{"role": "user", "content": marker}], + stream=True, + extra_body={"guardrails": [guardrail]}, + ): + if event.type == "content_block_delta" and event.delta.type == "text_delta": + yield "text", event.delta.text + + +async def _async_anthropic_events( + client: anthropic.AsyncAnthropic, marker: str, guardrail: str +) -> AsyncIterator[SdkEvent]: + async for event in await client.messages.create( + model=MODELS["messages"], + max_tokens=64, + messages=[{"role": "user", "content": marker}], + stream=True, + extra_body={"guardrails": [guardrail]}, + ): + if event.type == "content_block_delta" and event.delta.type == "text_delta": + yield "text", event.delta.text + + +def _sync_sdk(gateway: Gateway, endpoint: Endpoint, marker: str, guardrail: str) -> SdkOutcome: + origin: Final = str(gateway.client.base_url).rstrip("/") + + def events() -> Iterator[SdkEvent]: + try: + with httpx.Client(timeout=30, trust_env=False) as transport: + if endpoint == "messages": + with anthropic.Anthropic( + api_key=gateway.key, base_url=origin, max_retries=0, http_client=transport + ) as client: + yield from _anthropic_events(client, marker, guardrail) + return + with openai.OpenAI( + api_key=gateway.key, base_url=f"{origin}/v1", max_retries=0, http_client=transport + ) as client: + yield from _openai_events(client, endpoint, marker, guardrail) + except (openai.APIStatusError, anthropic.APIStatusError) as error: + yield "status", str(error.status_code) + except (openai.APIError, anthropic.APIError) as error: + yield "error", error.message + + return _outcome(tuple(events())) + + +def _async_sdk(gateway: Gateway, endpoint: Endpoint, marker: str, guardrail: str) -> SdkOutcome: + origin: Final = str(gateway.client.base_url).rstrip("/") + + async def events() -> AsyncIterator[SdkEvent]: + try: + async with httpx.AsyncClient(timeout=30, trust_env=False) as transport: + if endpoint == "messages": + async with anthropic.AsyncAnthropic( + api_key=gateway.key, base_url=origin, max_retries=0, http_client=transport + ) as client: + async for event in _async_anthropic_events(client, marker, guardrail): + yield event + return + async with openai.AsyncOpenAI( + api_key=gateway.key, base_url=f"{origin}/v1", max_retries=0, http_client=transport + ) as client: + async for event in _async_openai_events(client, endpoint, marker, guardrail): + yield event + except (openai.APIStatusError, anthropic.APIStatusError) as error: + yield "status", str(error.status_code) + except (openai.APIError, anthropic.APIError) as error: + yield "error", error.message + + async def collect() -> tuple[SdkEvent, ...]: + return tuple([event async for event in events()]) + + return _outcome(asyncio.run(collect())) + + +@pytest.mark.parametrize("client", ["sync", "async"]) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_sdk_clients_receive_no_text_before_a_buffered_block(rig: Rig, endpoint: Endpoint, client: Client) -> None: + marker: Final = _new_marker("block") + outcome: Final = {"sync": _sync_sdk, "async": _async_sdk}[client]( + rig.gateway, endpoint, marker, rig.names[("noma_v2", "buffered")] + ) + assert outcome.text == "", outcome + if endpoint == "responses": + assert outcome.failed and outcome.status is None, outcome + else: + assert outcome.status == 400, outcome + assert [scan.text for scan in rig.noma.for_marker(marker)] == [_full_text(marker)] + + +@pytest.mark.parametrize("variant", ["buffered", "invalid", "unset"]) +def test_stored_guardrail_info_reports_the_settings_as_stored(rig: Rig, variant: Variant) -> None: + info: Final = rig.gateway.get(f"/guardrails/{rig.stored[variant]}/info") + params: Final = object_value(info["litellm_params"]) + assert {key: params.get(key) for key in SETTINGS[variant]} == dict(SETTINGS[variant]), params + + +def _on_one_worker(rig: Rig, pid: int, guardrail: str) -> tuple[Streamed, ...]: + """Two blocked-prompt streams answered by worker `pid` alone, every other worker suspended meanwhile.""" + with ( + ExitStack() as paused, + httpx.Client( + base_url=str(rig.gateway.client.base_url), + timeout=15, + trust_env=False, + limits=httpx.Limits(max_keepalive_connections=0), + ) as client, + ): + for other in sorted(rig.proxy.worker_pids() - {pid}): + paused.enter_context(rig.proxy.suspended(other)) + alone: Final = Gateway(client, rig.gateway.key, rig.gateway.upstream_url) + return tuple(_send(rig, "chat", guardrail, "block", via=alone) for _ in range(2)) + + +def _converged_on(rig: Rig, pid: int, guardrail: str, satisfied: Callable[[Streamed], bool]) -> tuple[Streamed, ...]: + return eventually( + lambda: _on_one_worker(rig, pid, guardrail), + lambda attempts: all(satisfied(attempt) for attempt in attempts), + seconds=120, + ) + + +def test_patching_a_stored_guardrail_applies_the_setting_on_every_worker(rig: Rig) -> None: + name: Final = f"stored-patched-{uuid.uuid4().hex[:8]}" + workers: Final = sorted(eventually(rig.proxy.worker_pids, lambda found: len(found) == 2, seconds=30)) + identity: Final = _store(rig.gateway, rig.noma_url, name, "default") + try: + before: Final = tuple( + chain.from_iterable( + _converged_on(rig, pid, name, lambda attempt: len(attempt.scans) > 0) for pid in workers + ) + ) + assert all(SECRET in attempt.content for attempt in before), [attempt.response.text for attempt in before] + + patched: Final = rig.gateway.request( + "PATCH", f"/guardrails/{identity}", {"litellm_params": {"streaming_buffer_until_moderated": True}} + ) + assert patched.status_code == 200, patched.text + after: Final = tuple( + chain.from_iterable( + _converged_on(rig, pid, name, lambda attempt: attempt.response.status_code == 400) for pid in workers + ) + ) + assert all(attempt.content == "" and SECRET not in attempt.response.text for attempt in after), [ + attempt.response.text for attempt in after + ] + finally: + deleted: Final = rig.gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def test_burst_fails_closed_while_noma_is_down_and_recovers_after(rig: Rig) -> None: + guardrail: Final = rig.names[("noma_v2", "buffered")] + cases: Final = tuple( + (endpoint, action) + for endpoint in ENDPOINTS + for action in cast(tuple[Action, ...], ("pass", "block")) + for _ in range(5) + ) + rig.noma.down.set() + try: + down: Final = _burst(rig, cases, guardrail) + finally: + rig.noma.down.clear() + assert [streamed.response.text for streamed in down if streamed.content or SECRET in streamed.response.text] == [] + assert [streamed.scans for streamed in down if len(streamed.scans) != 1] == [] + + up: Final = _burst(rig, cases, guardrail) + delivered: Final = tuple(streamed for streamed in up if _action(streamed.marker) == "pass") + blocked: Final = tuple(streamed for streamed in up if _action(streamed.marker) == "block") + assert [s.response.text for s in delivered if s.content != _full_text(s.marker)] == [] + assert [ + s.response.text for s in blocked if s.content or SECRET in s.response.text or BLOCKED not in s.response.text + ] == [] + assert [streamed.scans for streamed in up if len(streamed.scans) != 1] == [] + + +def _first_text_seconds(rig: Rig, endpoint: Endpoint, guardrail: str) -> tuple[str, float | None, str]: + marker: Final = _new_marker("slow") + started: Final = time.monotonic() + with rig.gateway.client.stream( + "POST", + PATHS[endpoint], + json=_body(endpoint, marker, guardrail), + headers={"Authorization": f"Bearer {rig.gateway.key}"}, + timeout=30, + ) as response: + lines: Final = tuple((time.monotonic() - started, line) for line in response.iter_lines()) + texts: Final = tuple((elapsed, _line_text(line, endpoint)) for elapsed, line in lines) + return marker, next((elapsed for elapsed, text in texts if text), None), "".join(text for _, text in texts) + + +def test_slow_noma_holds_every_stream_until_its_verdict(rig: Rig) -> None: + guardrail: Final = rig.names[("noma_v2", "buffered")] + with ThreadPoolExecutor(max_workers=18) as pool: + streams: Final = tuple(pool.map(lambda endpoint: _first_text_seconds(rig, endpoint, guardrail), ENDPOINTS * 6)) + assert [stream for stream in streams if stream[2] != _full_text(stream[0])] == [] + assert [stream for stream in streams if stream[1] is None or stream[1] < SLOW_SECONDS] == [] + assert [marker for marker, _, _ in streams if len(rig.noma.for_marker(marker)) != 1] == [] + + +def test_replaced_workers_keep_the_streaming_settings(rig: Rig) -> None: + guardrail: Final = rig.names[("noma_v2", "buffered")] + originals: Final = eventually(rig.proxy.worker_pids, lambda found: len(found) == 2, seconds=30) + with ThreadPoolExecutor(max_workers=8) as pool: + inflight: Final = tuple( + pool.submit(_attempt, rig, ENDPOINTS[index % 3], guardrail, "block") for index in range(16) + ) + rig.proxy.kill_worker(min(originals)) + during: Final = tuple(future.result() for future in inflight) + rig.proxy.kill_worker(max(originals)) + assert [text for text in during if SECRET in text] == [] + assert rig.proxy.worker_pids().isdisjoint(originals), rig.proxy.worker_pids() + + after: Final = tuple( + _stream(rig, endpoint, "noma_v2", "buffered", "block", BUFFERED_BLOCK_STATUS[endpoint]) + for endpoint in ENDPOINTS + for _ in range(4) + ) + assert [streamed.response.text for streamed in after if streamed.content or SECRET in streamed.response.text] == [] + assert [streamed.scans for streamed in after if len(streamed.scans) != 1] == [] + + +@pytest.mark.parametrize("spelling", SPELLINGS) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_buffer_until_moderated_delivers_the_masked_text(rig: Rig, endpoint: Endpoint, spelling: Spelling) -> None: + pytest.skip("BUG: LIT-9559 a buffered stream releases the original chunks instead of the guardrail's masked text") + unbuffered: Final = _stream(rig, endpoint, spelling, "default", "mask") + assert SECRET in unbuffered.content, unbuffered.response.text + + buffered: Final = _stream(rig, endpoint, spelling, "buffered", "mask") + assert buffered.content == _full_text(buffered.marker).replace(SECRET, MASK), buffered.response.text + assert SECRET not in buffered.response.text, buffered.response.text diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/noma/test_noma_v2.py b/tests/unit/proxy/guardrails/guardrail_hooks/noma/test_noma_v2.py index 6e536f95251..8aca616faed 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/noma/test_noma_v2.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/noma/test_noma_v2.py @@ -1,17 +1,30 @@ import json +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import TypeAlias import httpx import pytest import respx import litellm +from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.proxy.guardrails.guardrail_hooks.noma import ( NomaV2Guardrail, guardrail_initializer_registry, ) +from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails +from litellm.proxy.guardrails.guardrail_registry import InMemoryGuardrailHandler from litellm.types.guardrails import LitellmParams _API_BASE = "https://noma.example.test" +_STREAMING_KEYS = ( + "streaming_buffer_until_moderated", + "streaming_end_of_stream_only", + "streaming_sampling_rate", + "streaming_buffer_release_on_scan", +) @pytest.fixture(autouse=True) @@ -97,3 +110,111 @@ async def test_positional_args_keep_their_meaning_after_gateway_name_was_added(r await guardrail.apply_guardrail( inputs={"texts": ["hello"]}, request_data={"metadata": {}}, input_type="request" ) + + +_Loader: TypeAlias = Callable[[Path, Mapping[str, object]], CustomGuardrail | None] +_SUBCLASS_MODULE = ( + "from litellm.proxy.guardrails.guardrail_hooks.noma.noma_v2 import NomaV2Guardrail\n\n\n" + "class SubclassedNoma(NomaV2Guardrail):\n" + " pass\n" +) + + +def _initializer(guardrail_type: str, fixed: Mapping[str, object]) -> _Loader: + def load(_tmp_path: Path, params: Mapping[str, object]) -> CustomGuardrail | None: + litellm_params = LitellmParams( + guardrail=guardrail_type, mode="post_call", api_base=_API_BASE, **fixed, **params + ) + return guardrail_initializer_registry[guardrail_type](litellm_params, {"guardrail_name": "noma-guard"}) + + return load + + +def _custom_subclass(tmp_path: Path, params: Mapping[str, object]) -> CustomGuardrail | None: + (tmp_path / "noma_subclass.py").write_text(_SUBCLASS_MODULE) + guardrail_type = "noma_subclass.SubclassedNoma" + return InMemoryGuardrailHandler().initialize_custom_guardrail( + guardrail={"guardrail_name": "noma-guard"}, + guardrail_type=guardrail_type, + litellm_params=LitellmParams(guardrail=guardrail_type, mode="post_call", api_base=_API_BASE, **params), + config_file_path=str(tmp_path / "config.yaml"), + ) + + +_LOADERS = pytest.mark.parametrize( + "load", + [_initializer("noma_v2", {}), _initializer("noma", {"use_v2": True}), _custom_subclass], + ids=["noma_v2", "noma-use_v2", "custom-subclass"], +) + + +def _streaming_flags(load: _Loader, tmp_path: Path, params: Mapping[str, object]) -> list[object]: + guardrail = load(tmp_path, params) + return [UnifiedLLMGuardrails().resolve_streaming_flag(guardrail, key, "hook-default") for key in _STREAMING_KEYS] + + +@_LOADERS +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ( + { + "streaming_buffer_until_moderated": True, + "streaming_end_of_stream_only": True, + "streaming_sampling_rate": 3, + "streaming_buffer_release_on_scan": True, + }, + [True, True, 3, True], + ), + ( + { + "streaming_buffer_until_moderated": False, + "streaming_end_of_stream_only": False, + "streaming_sampling_rate": 1, + "streaming_buffer_release_on_scan": False, + }, + [False, False, 1, False], + ), + ({"streaming_sampling_rate": 7}, ["hook-default", "hook-default", 7, "hook-default"]), + ( + {"streaming_buffer_until_moderated": "false", "streaming_sampling_rate": "2"}, + [False, "hook-default", 2, "hook-default"], + ), + ({}, ["hook-default"] * 4), + ], + ids=["all-on", "all-off", "one-key", "string-values", "unset"], +) +def test_streaming_settings_in_guardrail_config_reach_the_streaming_hook( + load: _Loader, tmp_path: Path, configured: dict[str, object], expected: list[object] +) -> None: + assert _streaming_flags(load, tmp_path, configured) == expected + + +@_LOADERS +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ({"streaming_sampling_rate": 0}, ["hook-default"] * 4), + ({"streaming_sampling_rate": -1}, ["hook-default"] * 4), + ({"streaming_sampling_rate": "often"}, ["hook-default"] * 4), + ({"streaming_buffer_until_moderated": "sometimes"}, ["hook-default"] * 4), + ( + {"streaming_buffer_until_moderated": True, "streaming_sampling_rate": 0}, + [True, "hook-default", "hook-default", "hook-default"], + ), + ], + ids=["rate-zero", "rate-negative", "rate-word", "buffer-word", "valid-key-kept"], +) +def test_invalid_streaming_value_is_ignored_and_the_guardrail_still_loads( + load: _Loader, + tmp_path: Path, + configured: dict[str, object], + expected: list[object], + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + assert _streaming_flags(load, tmp_path, configured) == expected + warnings = [record.getMessage() for record in caplog.records if "not a valid value" in record.getMessage()] + assert warnings, caplog.text + assert all(message.startswith("Noma v2 guardrail noma-guard: ignoring ") for message in warnings), warnings