mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge f70e996a7b into 969161501a
This commit is contained in:
commit
aba5ed9067
6 changed files with 527 additions and 35 deletions
|
|
@ -5,18 +5,26 @@ This module provides guardrail translation support for OpenAI's text completion
|
|||
The handler processes the 'prompt' parameter for guardrails.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
StreamingScanKey,
|
||||
StreamTransformSink,
|
||||
)
|
||||
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices, TextCompletionResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
|
||||
from litellm.types.utils import TextCompletionResponse
|
||||
|
||||
|
||||
class OpenAITextCompletionHandler(BaseTranslation):
|
||||
|
|
@ -32,6 +40,9 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
- A list of strings (for batch completions)
|
||||
"""
|
||||
|
||||
delivers_ended_stream_rewrites = True
|
||||
assembles_streamed_response = True
|
||||
|
||||
async def process_input_messages(
|
||||
self,
|
||||
data: dict,
|
||||
|
|
@ -125,7 +136,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
request_data: dict | None = None,
|
||||
request_data: dict[str, object] | None = None,
|
||||
) -> "TextCompletionResponse":
|
||||
"""
|
||||
Process output response by applying guardrails to completion text.
|
||||
|
|
@ -162,11 +173,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
|
||||
# Add user API key metadata with prefixed keys
|
||||
if "litellm_metadata" not in request_data:
|
||||
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
|
||||
if user_metadata:
|
||||
request_data["litellm_metadata"] = user_metadata
|
||||
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
|
||||
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
# Include model information from the response if available
|
||||
|
|
@ -195,3 +202,123 @@ class OpenAITextCompletionHandler(BaseTranslation):
|
|||
)
|
||||
|
||||
return response
|
||||
|
||||
async def process_output_streaming_response(
|
||||
self,
|
||||
responses_so_far: list[object], # mutable-ok: rewrites the caller's buffered chunks in place
|
||||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
request_data: dict[str, object] | None = None, # mutable-ok: the hooks share and mutate the request payload
|
||||
stream_transform_sink: StreamTransformSink | None = None,
|
||||
deliver_ended_stream_rewrites: bool = False,
|
||||
) -> Sequence[object]:
|
||||
chunks: Final = _text_completion_chunks(responses_so_far)
|
||||
if not self._check_streaming_has_ended(chunks):
|
||||
return responses_so_far
|
||||
assembled: Final = self._assemble_ended_stream(chunks)
|
||||
pre_guardrail_texts: Final = _choice_texts(assembled)
|
||||
await self.process_output_response(
|
||||
response=assembled,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=request_data,
|
||||
)
|
||||
if not deliver_ended_stream_rewrites:
|
||||
return responses_so_far
|
||||
rewrites: Final = tuple(
|
||||
after for before, after in zip(pre_guardrail_texts, _choice_texts(assembled)) if after != before
|
||||
)
|
||||
for choice_index, text in rewrites:
|
||||
self._write_choice_text(chunks, choice_index, text)
|
||||
return responses_so_far
|
||||
|
||||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
chunks: Final = _text_completion_chunks(responses_so_far)
|
||||
if not self._check_streaming_has_ended(chunks):
|
||||
return None
|
||||
assembled: Final = self._assemble_ended_stream(chunks)
|
||||
return StreamingScanKey(texts=tuple(text for _, text in _choice_texts(assembled)), stream_ended=True)
|
||||
|
||||
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
|
||||
chunks: Final = _text_completion_chunks(responses_so_far)
|
||||
streamed_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
|
||||
finished_indices: Final = frozenset(choice.index for choice in streamed_choices if choice.finish_reason)
|
||||
unfinished_indices: Final = tuple(
|
||||
dict.fromkeys(choice.index for choice in streamed_choices if choice.index not in finished_indices)
|
||||
)
|
||||
if not unfinished_indices:
|
||||
return tuple(responses_so_far)
|
||||
terminator: Final = TextCompletionResponse(
|
||||
id=chunks[0].id,
|
||||
model=chunks[0].model,
|
||||
object="text_completion",
|
||||
choices=[TextChoices(index=index, text="", finish_reason="length") for index in unfinished_indices],
|
||||
)
|
||||
return (*responses_so_far, terminator)
|
||||
|
||||
def build_stream_error_items(
|
||||
self,
|
||||
exc: "HTTPException",
|
||||
responses_so_far: Sequence[object] | None = None,
|
||||
) -> Sequence[bytes] | None:
|
||||
from litellm.proxy.common_request_processing import sse_error_payload
|
||||
|
||||
_, error_obj = sse_error_payload(exc)
|
||||
return (f'data: {{"error": {json.dumps(error_obj)}}}\n\n'.encode(),)
|
||||
|
||||
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
|
||||
streamed_choices: Final = chain.from_iterable(stream_item_items(item, "choices") for item in responses_so_far)
|
||||
return any(stream_item_field(choice, "finish_reason") is not None for choice in streamed_choices)
|
||||
|
||||
@staticmethod
|
||||
def _assemble_ended_stream(responses_so_far: Sequence[TextCompletionResponse]) -> TextCompletionResponse:
|
||||
streamed_choices: Final = tuple(chain.from_iterable(item.choices for item in responses_so_far))
|
||||
choice_indices: Final = tuple(dict.fromkeys(choice.index for choice in streamed_choices))
|
||||
assembled_choices: Final = [
|
||||
TextChoices(
|
||||
index=index,
|
||||
text="".join(_choice_text(choice) for choice in streamed_choices if choice.index == index),
|
||||
finish_reason=next(
|
||||
(
|
||||
choice.finish_reason
|
||||
for choice in streamed_choices
|
||||
if choice.index == index and choice.finish_reason
|
||||
),
|
||||
None,
|
||||
),
|
||||
)
|
||||
for index in choice_indices
|
||||
]
|
||||
first: Final = responses_so_far[0]
|
||||
return TextCompletionResponse(
|
||||
id=first.id,
|
||||
created=first.created,
|
||||
model=first.model,
|
||||
object="text_completion",
|
||||
choices=assembled_choices,
|
||||
usage=next((item.usage for item in reversed(responses_so_far) if item.usage is not None), None),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _write_choice_text(responses_so_far: Sequence[TextCompletionResponse], choice_index: int, text: str) -> None:
|
||||
streamed_choices: Final = chain.from_iterable(item.choices for item in responses_so_far)
|
||||
head, *tail = (choice for choice in streamed_choices if choice.index == choice_index)
|
||||
head.text = text
|
||||
for choice in tail:
|
||||
if isinstance(stream_item_field(choice, "text"), str):
|
||||
choice.text = ""
|
||||
|
||||
|
||||
def _text_completion_chunks(responses_so_far: Sequence[object]) -> tuple[TextCompletionResponse, ...]:
|
||||
return tuple(item for item in responses_so_far if isinstance(item, TextCompletionResponse))
|
||||
|
||||
|
||||
def _choice_text(choice: TextChoices) -> str:
|
||||
text: Final = stream_item_field(choice, "text")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
||||
|
||||
def _choice_texts(response: TextCompletionResponse) -> tuple[tuple[int, str], ...]:
|
||||
return tuple((choice.index, _choice_text(choice)) for choice in response.choices)
|
||||
|
|
|
|||
|
|
@ -82,6 +82,21 @@ def _as_plain_dict(item: object) -> Mapping[str, object]:
|
|||
return cast("Mapping[str, object]", item) # cast-ok: wire rows are message/tool-call dicts
|
||||
|
||||
|
||||
_STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _str_keyed(value: object) -> Mapping[str, object]:
|
||||
return _STR_KEYED_MAPPING.validate_python(value) if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def _proxy_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]:
|
||||
litellm_metadata: Final = request_data.get("litellm_metadata")
|
||||
if litellm_metadata is not None:
|
||||
return _str_keyed(litellm_metadata)
|
||||
metadata: Final = _str_keyed(request_data.get("metadata"))
|
||||
return {key: value for key, value in metadata.items() if key.startswith("user_api_key_")}
|
||||
|
||||
|
||||
class GraySwanGuardrailMissingSecrets(Exception):
|
||||
"""Raised when the Gray Swan API key is missing."""
|
||||
|
||||
|
|
@ -625,10 +640,7 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
|
||||
from litellm.llms import load_guardrail_translation_mappings
|
||||
|
||||
litellm_metadata: Final = request_data.get("litellm_metadata")
|
||||
request_route: Final = (
|
||||
litellm_metadata.get("user_api_key_request_route") if isinstance(litellm_metadata, Mapping) else None
|
||||
)
|
||||
request_route: Final = _proxy_metadata(request_data).get("user_api_key_request_route")
|
||||
route_call_types: Final = get_call_types_for_route(request_route) if isinstance(request_route, str) else None
|
||||
call_type: Final = (
|
||||
(route_call_types[0].value if route_call_types else None)
|
||||
|
|
@ -698,13 +710,12 @@ class GraySwanGuardrail(CustomGuardrail):
|
|||
|
||||
inbound_headers: Final = self._extract_inbound_headers(request_data, logging_obj)
|
||||
|
||||
litellm_metadata: Final = request_data.get("litellm_metadata")
|
||||
cleaned_litellm_metadata: Final = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {}
|
||||
if inbound_headers:
|
||||
existing_headers: Final = cleaned_litellm_metadata.get("headers")
|
||||
cleaned_litellm_metadata["headers"] = (
|
||||
{**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers
|
||||
)
|
||||
proxy_metadata: Final = _proxy_metadata(request_data)
|
||||
cleaned_litellm_metadata: Final = (
|
||||
{**proxy_metadata, "headers": {**_str_keyed(proxy_metadata.get("headers")), **inbound_headers}}
|
||||
if inbound_headers
|
||||
else proxy_metadata
|
||||
)
|
||||
if cleaned_litellm_metadata:
|
||||
sanitized: Final = _DECODED_JSON.validate_python(
|
||||
safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={})
|
||||
|
|
|
|||
|
|
@ -27,7 +27,12 @@ _PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
|
|||
_PROVIDER_KEY: Final = "synthetic-provider-key"
|
||||
_GUARDRAIL: Final = "audit-text-rebuild-tool-permission"
|
||||
_TEXT_MODEL: Final = f"audit-text-instruct-{uuid.uuid4().hex}"
|
||||
_GOVERNED_MODEL: Final = f"audit-text-governed-{uuid.uuid4().hex}"
|
||||
_KEYWORD_FILTER: Final = "audit-text-keyword-filter"
|
||||
_POLICY: Final = "audit-text-post-call-governance"
|
||||
_PIECES: Final = ("Hello", " streamed", " world")
|
||||
_MASKED_PIECES: Final = ("A pers", "im", "mon tart")
|
||||
_BLOCKED_PIECES: Final = ("A kum", "quat tart")
|
||||
_PROMPT_TOKENS: Final = 11
|
||||
_COMPLETION_TOKENS: Final = 3
|
||||
_USAGE: Final[dict[str, JsonValue]] = {
|
||||
|
|
@ -97,6 +102,14 @@ def _text_choice(text: str, finish_reason: str | None) -> dict[str, JsonValue]:
|
|||
return {"text": text, "index": 0, "logprobs": None, "finish_reason": finish_reason}
|
||||
|
||||
|
||||
def _pieces_for(identity: str) -> tuple[str, ...]:
|
||||
if identity.startswith("cmpl-mask-"):
|
||||
return _MASKED_PIECES
|
||||
if identity.startswith("cmpl-block-"):
|
||||
return _BLOCKED_PIECES
|
||||
return _PIECES
|
||||
|
||||
|
||||
def _respond(request: Request) -> Reply:
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
assert body.get("stream") is True, body
|
||||
|
|
@ -106,7 +119,7 @@ def _respond(request: Request) -> Reply:
|
|||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
*(_text_frame(identity, [_text_choice(piece, None)]) for piece in _PIECES),
|
||||
*(_text_frame(identity, [_text_choice(piece, None)]) for piece in _pieces_for(identity)),
|
||||
_text_frame(identity, [_text_choice("", "length")]),
|
||||
_text_frame(identity, [], usage=_USAGE),
|
||||
b"data: [DONE]\n\n",
|
||||
|
|
@ -114,20 +127,25 @@ def _respond(request: Request) -> Reply:
|
|||
)
|
||||
|
||||
|
||||
def _text_deployment(model_name: str, wire: Wire) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
||||
"api_base": f"{wire.url}/v1",
|
||||
"api_key": _PROVIDER_KEY,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _config(directory: Path, wire: Wire) -> Path:
|
||||
base: Final = object_value(yaml.safe_load(_PROXY_CONFIG.read_text()))
|
||||
config: Final = {
|
||||
**base,
|
||||
"model_list": [
|
||||
*list_value(base["model_list"]),
|
||||
{
|
||||
"model_name": _TEXT_MODEL,
|
||||
"litellm_params": {
|
||||
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
|
||||
"api_base": f"{wire.url}/v1",
|
||||
"api_key": _PROVIDER_KEY,
|
||||
},
|
||||
},
|
||||
_text_deployment(_TEXT_MODEL, wire),
|
||||
_text_deployment(_GOVERNED_MODEL, wire),
|
||||
],
|
||||
"guardrails": [
|
||||
{
|
||||
|
|
@ -139,8 +157,30 @@ def _config(directory: Path, wire: Wire) -> Path:
|
|||
"default_action": "allow",
|
||||
"rules": [],
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"guardrail_name": _KEYWORD_FILTER,
|
||||
"litellm_params": {
|
||||
"guardrail": "litellm_content_filter",
|
||||
"mode": "post_call",
|
||||
"default_on": False,
|
||||
"blocked_words": [
|
||||
{"keyword": "persimmon", "action": "MASK"},
|
||||
{"keyword": "kumquat", "action": "BLOCK"},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
"policies": {
|
||||
_POLICY: {
|
||||
"guardrails": {"add": [_KEYWORD_FILTER]},
|
||||
"pipeline": {
|
||||
"mode": "post_call",
|
||||
"steps": [{"guardrail": _KEYWORD_FILTER, "on_pass": "allow", "on_fail": "block"}],
|
||||
},
|
||||
}
|
||||
},
|
||||
"policy_attachments": [{"policy": _POLICY, "models": [_GOVERNED_MODEL]}],
|
||||
}
|
||||
path: Final = directory / f"text-rebuild-{uuid.uuid4().hex}.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
|
|
@ -163,14 +203,14 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
|
|||
yield _Rig(wire, owned)
|
||||
|
||||
|
||||
def _stream(rig: _Rig, identity: str) -> _Stream:
|
||||
def _stream(rig: _Rig, identity: str, model: str = _TEXT_MODEL) -> _Stream:
|
||||
gateway: Final = rig.proxy.gateway
|
||||
with gateway.client.stream(
|
||||
"POST",
|
||||
"/v1/completions",
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
json={
|
||||
"model": _TEXT_MODEL,
|
||||
"model": model,
|
||||
"prompt": identity,
|
||||
"max_tokens": 16,
|
||||
"stream": True,
|
||||
|
|
@ -212,6 +252,37 @@ def test_guarded_text_completion_stream_reaches_the_caller_whole_with_its_usage(
|
|||
assert _GUARDRAIL in list_value(object_value(row["metadata"])["applied_guardrails"]), row
|
||||
|
||||
|
||||
def test_post_call_pipeline_masks_a_streamed_text_completion(rig: _Rig) -> None:
|
||||
identity: Final = f"cmpl-mask-{uuid.uuid4().hex}"
|
||||
|
||||
stream: Final = _stream(rig, identity, _GOVERNED_MODEL)
|
||||
|
||||
assert stream.status == 200, stream.body
|
||||
assert stream.body.rstrip().endswith(b"data: [DONE]"), stream.body[-400:]
|
||||
assert stream.text == "A [KEYWORD_REDACTED] tart", stream.frames
|
||||
assert b"persimmon" not in stream.body, stream.body
|
||||
assert stream.finish_reasons == ("length",), stream.frames
|
||||
assert [(usage["prompt_tokens"], usage["completion_tokens"]) for usage in stream.usages] == [
|
||||
(_PROMPT_TOKENS, _COMPLETION_TOKENS)
|
||||
], stream.frames
|
||||
(row,) = eventually(
|
||||
lambda: rig.proxy.gateway.rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (identity,)),
|
||||
lambda rows: len(rows) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert _KEYWORD_FILTER in list_value(object_value(row["metadata"])["applied_guardrails"]), row
|
||||
|
||||
|
||||
def test_post_call_pipeline_blocks_a_streamed_text_completion(rig: _Rig) -> None:
|
||||
identity: Final = f"cmpl-block-{uuid.uuid4().hex}"
|
||||
|
||||
stream: Final = _stream(rig, identity, _GOVERNED_MODEL)
|
||||
|
||||
assert stream.status == 400, stream.body
|
||||
assert b"Content blocked: keyword 'kumquat' detected" in stream.body, stream.body
|
||||
assert b"quat tart" not in stream.body, stream.body
|
||||
|
||||
|
||||
def _together_chunk(choices: list[JsonValue], **extra: JsonValue) -> bytes:
|
||||
return _frame(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from typing import List, Optional, Tuple
|
|||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
|
|
@ -13,7 +14,11 @@ from litellm.llms import get_guardrail_translation_mapping
|
|||
from litellm.llms.openai.completion.guardrail_translation.handler import (
|
||||
OpenAITextCompletionHandler,
|
||||
)
|
||||
from litellm.types.utils import CallTypes, TextChoices, TextCompletionResponse
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
|
||||
from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails
|
||||
from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
|
||||
from litellm.types.utils import CallTypes, TextChoices, TextCompletionResponse, Usage
|
||||
|
||||
|
||||
class MockGuardrail(CustomGuardrail):
|
||||
|
|
@ -341,3 +346,143 @@ class TestPIIMaskingScenario:
|
|||
assert "[EMAIL_REDACTED]" in result["prompt"][1]
|
||||
assert "alice@example.com" not in result["prompt"][0]
|
||||
assert "bob@test.org" not in result["prompt"][1]
|
||||
|
||||
|
||||
class _RecordingKeywordFilter(ContentFilterGuardrail):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
guardrail_name="keyword-filter",
|
||||
blocked_words=[
|
||||
BlockedWord(keyword="persimmon", action=ContentFilterAction.MASK),
|
||||
BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK),
|
||||
],
|
||||
event_hook=GuardrailEventHooks.post_call,
|
||||
default_on=True,
|
||||
)
|
||||
self.streaming_sampling_rate = 1
|
||||
self.scanned_texts: tuple[tuple[str, ...], ...] = ()
|
||||
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
self.scanned_texts = (*self.scanned_texts, tuple(inputs.get("texts") or ()))
|
||||
return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
|
||||
|
||||
|
||||
def _text_chunk(index: int, text: str | None, finish_reason: str | None = None) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-stream",
|
||||
created=1589478378,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
object="text_completion",
|
||||
choices=[TextChoices(index=index, text=text, finish_reason=finish_reason)],
|
||||
)
|
||||
|
||||
|
||||
def _usage_chunk() -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-stream",
|
||||
created=1589478378,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
object="text_completion",
|
||||
choices=[],
|
||||
usage=Usage(prompt_tokens=3, completion_tokens=5, total_tokens=8),
|
||||
)
|
||||
|
||||
|
||||
def _delivered(chunks: list[TextCompletionResponse]) -> list[tuple[int, str | None, str | None]]:
|
||||
return [(choice.index, choice.text, choice.finish_reason) for chunk in chunks for choice in chunk.choices]
|
||||
|
||||
|
||||
async def _drive_flat_stream(guardrail: CustomGuardrail, chunks: list[TextCompletionResponse]) -> list[object]:
|
||||
async def upstream():
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
return [
|
||||
item
|
||||
async for item in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/completions"),
|
||||
response=upstream(),
|
||||
request_data={"guardrail_to_apply": guardrail, "model": "gpt-3.5-turbo-instruct"},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestStreamingOutputProcessing:
|
||||
@pytest.mark.asyncio
|
||||
async def test_ended_stream_delivers_each_choice_masked_text(self):
|
||||
guardrail = _RecordingKeywordFilter()
|
||||
chunks = [
|
||||
_text_chunk(0, "A pers"),
|
||||
_text_chunk(1, "kiwi "),
|
||||
_text_chunk(0, "im"),
|
||||
_text_chunk(1, "pie", "stop"),
|
||||
_text_chunk(0, "mon tart"),
|
||||
_text_chunk(0, None, "length"),
|
||||
_usage_chunk(),
|
||||
]
|
||||
|
||||
await OpenAITextCompletionHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=guardrail,
|
||||
request_data={},
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
assert guardrail.scanned_texts == (("A persimmon tart", "kiwi pie"),)
|
||||
assert _delivered(chunks) == [
|
||||
(0, "A [KEYWORD_REDACTED] tart", None),
|
||||
(1, "kiwi ", None),
|
||||
(0, "", None),
|
||||
(1, "pie", "stop"),
|
||||
(0, "", None),
|
||||
(0, None, "length"),
|
||||
]
|
||||
assert chunks[-1].usage.total_tokens == 8
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ended_stream_scanned_without_delivery_keeps_chunks_and_stores_the_assembled_response(self):
|
||||
chunks = [_text_chunk(0, "A pers"), _text_chunk(0, "immon tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
|
||||
request_data: dict[str, object] = {}
|
||||
|
||||
await OpenAITextCompletionHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=_RecordingKeywordFilter(), request_data=request_data
|
||||
)
|
||||
|
||||
assert _delivered(chunks) == [(0, "A pers", None), (0, "immon tart", None), (0, "", "stop")]
|
||||
assembled = request_data["response"]
|
||||
assert isinstance(assembled, TextCompletionResponse)
|
||||
assert [(c.text, c.finish_reason) for c in assembled.choices] == [("A [KEYWORD_REDACTED] tart", "stop")]
|
||||
assert assembled.usage.total_tokens == 8
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ended_stream_block_raises_the_guardrail_error(self):
|
||||
chunks = [_text_chunk(0, "A kum"), _text_chunk(0, "quat tart"), _text_chunk(0, "", "stop")]
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await OpenAITextCompletionHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=_RecordingKeywordFilter(), deliver_ended_stream_rewrites=True
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "kumquat" in str(exc_info.value.detail)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flat_stream_is_scanned_once_after_it_ends(self):
|
||||
guardrail = _RecordingKeywordFilter()
|
||||
chunks = [_text_chunk(0, "A pers"), _text_chunk(0, "immon tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
|
||||
|
||||
delivered = await _drive_flat_stream(guardrail, chunks)
|
||||
|
||||
assert guardrail.scanned_texts == (("A persimmon tart",),)
|
||||
assert len(delivered) == len(chunks)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flat_stream_block_ends_with_an_in_band_error_frame(self):
|
||||
chunks = [_text_chunk(0, "A kum"), _text_chunk(0, "quat tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
|
||||
|
||||
delivered = await _drive_flat_stream(_RecordingKeywordFilter(), chunks)
|
||||
|
||||
error_frame = delivered[-1]
|
||||
assert isinstance(error_frame, bytes)
|
||||
assert error_frame.startswith(b'data: {"error": ')
|
||||
assert b"kumquat" in error_frame
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import pytest
|
|||
from fastapi import HTTPException
|
||||
|
||||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.llms.openai.completion.guardrail_translation.handler import OpenAITextCompletionHandler
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_hooks.grayswan import grayswan as grayswan_module
|
||||
from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import (
|
||||
|
|
@ -12,6 +13,7 @@ from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import (
|
|||
GraySwanGuardrailAPIError,
|
||||
)
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
from litellm.types.utils import TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -884,3 +886,24 @@ async def test_pre_call_payload_unchanged() -> None:
|
|||
{"role": "user", "content": "second"},
|
||||
]
|
||||
assert "tools" not in payload
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_on_a_text_completion_forwards_only_the_key_and_team_metadata() -> None:
|
||||
guardrail = _post_call_guardrail()
|
||||
client = _CapturingClient()
|
||||
guardrail.async_handler = client
|
||||
|
||||
await OpenAITextCompletionHandler().process_output_response(
|
||||
response=TextCompletionResponse(
|
||||
id="cmpl-1", model="gpt-4.1-mini", choices=[TextChoices(index=0, text="hello", finish_reason="stop")]
|
||||
),
|
||||
guardrail_to_apply=guardrail,
|
||||
user_api_key_dict=UserAPIKeyAuth(team_id="team-456", key_alias="ci-key"),
|
||||
request_data={"model": "gpt-4.1-mini", "prompt": "say hello", "metadata": {"requester_note": "private"}},
|
||||
)
|
||||
|
||||
forwarded = client.calls[0]["json"]["litellm_metadata"]
|
||||
assert forwarded["user_api_key_team_id"] == "team-456"
|
||||
assert forwarded["user_api_key_key_alias"] == "ci-key"
|
||||
assert "requester_note" not in forwarded
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
|
|||
GuardrailPipeline,
|
||||
PipelineStep,
|
||||
)
|
||||
from litellm.types.utils import TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
|
|
@ -1886,7 +1887,7 @@ def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"request_route",
|
||||
["/v1/completions", "/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"],
|
||||
["/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"],
|
||||
)
|
||||
def test_streamable_post_call_pipelines_keeps_legacy_hooks_off_routes_that_assemble_no_response(
|
||||
make_user_api_key_auth, monkeypatch, caplog, request_route
|
||||
|
|
@ -2036,6 +2037,120 @@ async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_every_cho
|
|||
assert _warnings(caplog) == []
|
||||
|
||||
|
||||
def _text_completion_chunk(text: str, finish_reason: str | None = None) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id="cmpl-1", model="m", choices=[TextChoices(index=0, text=text, finish_reason=finish_reason)]
|
||||
)
|
||||
|
||||
|
||||
def _rewritten_text_completion(response: Any) -> TextCompletionResponse:
|
||||
return TextCompletionResponse(
|
||||
id=response.id,
|
||||
model=response.model,
|
||||
choices=[TextChoices(index=0, text="[REWRITTEN] " + response.choices[0].text, finish_reason="stop")],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_a_text_completion_stream(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch, caplog
|
||||
):
|
||||
seen: Dict[str, Any] = {}
|
||||
monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, rewrite=_rewritten_text_completion)])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
|
||||
data = {**_post_call_pipeline_data(stream=True), "prompt": "hi"}
|
||||
chunks = [_text_completion_chunk("hello "), _text_completion_chunk("world"), _text_completion_chunk("", "stop")]
|
||||
auth = make_user_api_key_auth(request_route="/v1/completions")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
|
||||
await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True
|
||||
)
|
||||
delivered = [
|
||||
item
|
||||
async for item in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
|
||||
)
|
||||
]
|
||||
|
||||
assert isinstance(seen["response"], TextCompletionResponse)
|
||||
assert seen["response"].choices[0].text == "hello world"
|
||||
assert [(item.choices[0].text, item.choices[0].finish_reason) for item in delivered] == [
|
||||
("[REWRITTEN] hello world", None),
|
||||
("", None),
|
||||
("", "stop"),
|
||||
]
|
||||
assert data["metadata"]["applied_guardrails"] == ["gr-post"]
|
||||
assert _warnings(caplog) == []
|
||||
|
||||
|
||||
class _FlatBlockGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
if any("kumquat" in text for text in inputs.get("texts") or []):
|
||||
raise HTTPException(status_code=400, detail="flat guardrail blocked kumquat")
|
||||
return inputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_iterator_hook_delivers_a_flat_guardrail_block_on_a_text_completion_stream_under_a_pipeline(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
seen: Final[Dict[str, Any]] = {}
|
||||
flat: Final = _FlatBlockGuardrail(
|
||||
guardrail_name="flat-block", event_hook=GuardrailEventHooks.post_call, default_on=True
|
||||
)
|
||||
monkeypatch.setattr(litellm, "callbacks", [flat, _live_scan_guardrail(seen)])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
|
||||
data: Final = {**_post_call_pipeline_data(stream=True), "prompt": "hi"}
|
||||
chunks: Final = [
|
||||
_text_completion_chunk("A kum"),
|
||||
_text_completion_chunk("quat tart"),
|
||||
_text_completion_chunk("", "stop"),
|
||||
]
|
||||
auth: Final = make_user_api_key_auth(request_route="/v1/completions")
|
||||
|
||||
await proxy_logging.pre_call_hook(user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True)
|
||||
delivered: Final = [
|
||||
item
|
||||
async for item in proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
|
||||
)
|
||||
]
|
||||
|
||||
assert seen["texts"] == ["A kumquat tart"]
|
||||
assert isinstance(delivered[-1], bytes)
|
||||
assert json.loads(delivered[-1].decode().removeprefix("data: "))["error"]["message"] == (
|
||||
"flat guardrail blocked kumquat"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_iterator_hook_live_pipeline_scans_what_a_disconnected_text_completion_client_received(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
):
|
||||
seen: Final[Dict[str, Any]] = {}
|
||||
monkeypatch.setattr(litellm, "callbacks", [_live_scan_guardrail(seen)])
|
||||
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
|
||||
data: Final = {**_post_call_pipeline_data(step=_DETECT_ONLY_STEP, stream=True), "prompt": "hi"}
|
||||
chunks: Final = [
|
||||
_text_completion_chunk("hello "),
|
||||
_text_completion_chunk("world"),
|
||||
_text_completion_chunk("", "stop"),
|
||||
]
|
||||
auth: Final = make_user_api_key_auth(request_route="/v1/completions")
|
||||
|
||||
await proxy_logging.pre_call_hook(user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True)
|
||||
stream: Final = proxy_logging.async_post_call_streaming_iterator_hook(
|
||||
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
|
||||
)
|
||||
first: Final = await stream.__anext__()
|
||||
await stream.aclose()
|
||||
|
||||
assert first.choices[0].text == "hello "
|
||||
assert seen["texts"] == ["hello "]
|
||||
assert _post_call_statuses(data) == ["success"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_iterator_hook_releases_stream_untouched_when_legacy_hook_returns_none(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue