This commit is contained in:
Mateo Wang 2026-10-27 02:24:57 +00:00 • committed by GitHub
commit 7e4f75b5a0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 527 additions and 35 deletions

View file

@ -5,18 +5,26 @@ This module provides guardrail translation support for OpenAI's text completion
The handler processes the 'prompt' parameter for guardrails.
"""
from collections.abc import Mapping
import json
from collections.abc import Mapping, Sequence
from itertools import chain
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_proxy_logger
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
from litellm.types.utils import GenericGuardrailAPIInputs
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
StreamTransformSink,
)
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
from litellm.types.utils import GenericGuardrailAPIInputs, TextChoices, TextCompletionResponse
if TYPE_CHECKING:
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
from litellm.types.utils import TextCompletionResponse
class OpenAITextCompletionHandler(BaseTranslation):
@ -32,6 +40,9 @@ class OpenAITextCompletionHandler(BaseTranslation):
- A list of strings (for batch completions)
"""
delivers_ended_stream_rewrites = True
assembles_streamed_response = True
async def process_input_messages(
self,
data: dict,
@ -125,7 +136,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict | None = None,
request_data: dict[str, object] | None = None,
) -> "TextCompletionResponse":
"""
Process output response by applying guardrails to completion text.
@ -162,11 +173,7 @@ class OpenAITextCompletionHandler(BaseTranslation):
if "response" not in request_data:
request_data["response"] = response
# Add user API key metadata with prefixed keys
if "litellm_metadata" not in request_data:
user_metadata: Final = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
if user_metadata:
request_data["litellm_metadata"] = user_metadata
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
# Include model information from the response if available
@ -195,3 +202,123 @@ class OpenAITextCompletionHandler(BaseTranslation):
)
return response
async def process_output_streaming_response(
self,
responses_so_far: list[object], # mutable-ok: rewrites the caller's buffered chunks in place
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict[str, object] | None = None, # mutable-ok: the hooks share and mutate the request payload
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> Sequence[object]:
chunks: Final = _text_completion_chunks(responses_so_far)
if not self._check_streaming_has_ended(chunks):
return responses_so_far
assembled: Final = self._assemble_ended_stream(chunks)
pre_guardrail_texts: Final = _choice_texts(assembled)
await self.process_output_response(
response=assembled,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
)
if not deliver_ended_stream_rewrites:
return responses_so_far
rewrites: Final = tuple(
after for before, after in zip(pre_guardrail_texts, _choice_texts(assembled)) if after != before
)
for choice_index, text in rewrites:
self._write_choice_text(chunks, choice_index, text)
return responses_so_far
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
chunks: Final = _text_completion_chunks(responses_so_far)
if not self._check_streaming_has_ended(chunks):
return None
assembled: Final = self._assemble_ended_stream(chunks)
return StreamingScanKey(texts=tuple(text for _, text in _choice_texts(assembled)), stream_ended=True)
def released_stream_as_ended(self, responses_so_far: Sequence[object]) -> tuple[object, ...]:
chunks: Final = _text_completion_chunks(responses_so_far)
streamed_choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks))
finished_indices: Final = frozenset(choice.index for choice in streamed_choices if choice.finish_reason)
unfinished_indices: Final = tuple(
dict.fromkeys(choice.index for choice in streamed_choices if choice.index not in finished_indices)
)
if not unfinished_indices:
return tuple(responses_so_far)
terminator: Final = TextCompletionResponse(
id=chunks[0].id,
model=chunks[0].model,
object="text_completion",
choices=[TextChoices(index=index, text="", finish_reason="length") for index in unfinished_indices],
)
return (*responses_so_far, terminator)
def build_stream_error_items(
self,
exc: "HTTPException",
responses_so_far: Sequence[object] | None = None,
) -> Sequence[bytes] | None:
from litellm.proxy.common_request_processing import sse_error_payload
_, error_obj = sse_error_payload(exc)
return (f'data: {{"error": {json.dumps(error_obj)}}}\n\n'.encode(),)
def _check_streaming_has_ended(self, responses_so_far: Sequence[object]) -> bool:
streamed_choices: Final = chain.from_iterable(stream_item_items(item, "choices") for item in responses_so_far)
return any(stream_item_field(choice, "finish_reason") is not None for choice in streamed_choices)
@staticmethod
def _assemble_ended_stream(responses_so_far: Sequence[TextCompletionResponse]) -> TextCompletionResponse:
streamed_choices: Final = tuple(chain.from_iterable(item.choices for item in responses_so_far))
choice_indices: Final = tuple(dict.fromkeys(choice.index for choice in streamed_choices))
assembled_choices: Final = [
TextChoices(
index=index,
text="".join(_choice_text(choice) for choice in streamed_choices if choice.index == index),
finish_reason=next(
(
choice.finish_reason
for choice in streamed_choices
if choice.index == index and choice.finish_reason
),
None,
),
)
for index in choice_indices
]
first: Final = responses_so_far[0]
return TextCompletionResponse(
id=first.id,
created=first.created,
model=first.model,
object="text_completion",
choices=assembled_choices,
usage=next((item.usage for item in reversed(responses_so_far) if item.usage is not None), None),
)
@staticmethod
def _write_choice_text(responses_so_far: Sequence[TextCompletionResponse], choice_index: int, text: str) -> None:
streamed_choices: Final = chain.from_iterable(item.choices for item in responses_so_far)
head, *tail = (choice for choice in streamed_choices if choice.index == choice_index)
head.text = text
for choice in tail:
if isinstance(stream_item_field(choice, "text"), str):
choice.text = ""
def _text_completion_chunks(responses_so_far: Sequence[object]) -> tuple[TextCompletionResponse, ...]:
return tuple(item for item in responses_so_far if isinstance(item, TextCompletionResponse))
def _choice_text(choice: TextChoices) -> str:
text: Final = stream_item_field(choice, "text")
return text if isinstance(text, str) else ""
def _choice_texts(response: TextCompletionResponse) -> tuple[tuple[int, str], ...]:
return tuple((choice.index, _choice_text(choice)) for choice in response.choices)

View file

@ -82,6 +82,21 @@ def _as_plain_dict(item: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", item) # cast-ok: wire rows are message/tool-call dicts
_STR_KEYED_MAPPING: Final = TypeAdapter(Mapping[str, object])
def _str_keyed(value: object) -> Mapping[str, object]:
return _STR_KEYED_MAPPING.validate_python(value) if isinstance(value, Mapping) else {}
def _proxy_metadata(request_data: Mapping[str, object]) -> Mapping[str, object]:
litellm_metadata: Final = request_data.get("litellm_metadata")
if litellm_metadata is not None:
return _str_keyed(litellm_metadata)
metadata: Final = _str_keyed(request_data.get("metadata"))
return {key: value for key, value in metadata.items() if key.startswith("user_api_key_")}
class GraySwanGuardrailMissingSecrets(Exception):
"""Raised when the Gray Swan API key is missing."""
@ -625,10 +640,7 @@ class GraySwanGuardrail(CustomGuardrail):
from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
from litellm.llms import load_guardrail_translation_mappings
litellm_metadata: Final = request_data.get("litellm_metadata")
request_route: Final = (
litellm_metadata.get("user_api_key_request_route") if isinstance(litellm_metadata, Mapping) else None
)
request_route: Final = _proxy_metadata(request_data).get("user_api_key_request_route")
route_call_types: Final = get_call_types_for_route(request_route) if isinstance(request_route, str) else None
call_type: Final = (
(route_call_types[0].value if route_call_types else None)
@ -698,13 +710,12 @@ class GraySwanGuardrail(CustomGuardrail):
inbound_headers: Final = self._extract_inbound_headers(request_data, logging_obj)
litellm_metadata: Final = request_data.get("litellm_metadata")
cleaned_litellm_metadata: Final = dict(litellm_metadata) if isinstance(litellm_metadata, dict) else {}
if inbound_headers:
existing_headers: Final = cleaned_litellm_metadata.get("headers")
cleaned_litellm_metadata["headers"] = (
{**existing_headers, **inbound_headers} if isinstance(existing_headers, dict) else inbound_headers
)
proxy_metadata: Final = _proxy_metadata(request_data)
cleaned_litellm_metadata: Final = (
{**proxy_metadata, "headers": {**_str_keyed(proxy_metadata.get("headers")), **inbound_headers}}
if inbound_headers
else proxy_metadata
)
if cleaned_litellm_metadata:
sanitized: Final = _DECODED_JSON.validate_python(
safe_json_loads(safe_dumps(cleaned_litellm_metadata), default={})

View file

@ -27,7 +27,12 @@ _PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
_PROVIDER_KEY: Final = "synthetic-provider-key"
_GUARDRAIL: Final = "audit-text-rebuild-tool-permission"
_TEXT_MODEL: Final = f"audit-text-instruct-{uuid.uuid4().hex}"
_GOVERNED_MODEL: Final = f"audit-text-governed-{uuid.uuid4().hex}"
_KEYWORD_FILTER: Final = "audit-text-keyword-filter"
_POLICY: Final = "audit-text-post-call-governance"
_PIECES: Final = ("Hello", " streamed", " world")
_MASKED_PIECES: Final = ("A pers", "im", "mon tart")
_BLOCKED_PIECES: Final = ("A kum", "quat tart")
_PROMPT_TOKENS: Final = 11
_COMPLETION_TOKENS: Final = 3
_USAGE: Final[dict[str, JsonValue]] = {
@ -97,6 +102,14 @@ def _text_choice(text: str, finish_reason: str | None) -> dict[str, JsonValue]:
return {"text": text, "index": 0, "logprobs": None, "finish_reason": finish_reason}
def _pieces_for(identity: str) -> tuple[str, ...]:
if identity.startswith("cmpl-mask-"):
return _MASKED_PIECES
if identity.startswith("cmpl-block-"):
return _BLOCKED_PIECES
return _PIECES
def _respond(request: Request) -> Reply:
body: Final = JSON_OBJECT.validate_json(request.body)
assert body.get("stream") is True, body
@ -106,7 +119,7 @@ def _respond(request: Request) -> Reply:
return Reply(
content_type="text/event-stream",
chunks=(
*(_text_frame(identity, [_text_choice(piece, None)]) for piece in _PIECES),
*(_text_frame(identity, [_text_choice(piece, None)]) for piece in _pieces_for(identity)),
_text_frame(identity, [_text_choice("", "length")]),
_text_frame(identity, [], usage=_USAGE),
b"data: [DONE]\n\n",
@ -114,20 +127,25 @@ def _respond(request: Request) -> Reply:
)
def _text_deployment(model_name: str, wire: Wire) -> dict[str, JsonValue]:
return {
"model_name": model_name,
"litellm_params": {
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
"api_base": f"{wire.url}/v1",
"api_key": _PROVIDER_KEY,
},
}
def _config(directory: Path, wire: Wire) -> Path:
base: Final = object_value(yaml.safe_load(_PROXY_CONFIG.read_text()))
config: Final = {
**base,
"model_list": [
*list_value(base["model_list"]),
{
"model_name": _TEXT_MODEL,
"litellm_params": {
"model": "text-completion-openai/gpt-3.5-turbo-instruct",
"api_base": f"{wire.url}/v1",
"api_key": _PROVIDER_KEY,
},
},
_text_deployment(_TEXT_MODEL, wire),
_text_deployment(_GOVERNED_MODEL, wire),
],
"guardrails": [
{
@ -139,8 +157,30 @@ def _config(directory: Path, wire: Wire) -> Path:
"default_action": "allow",
"rules": [],
},
}
},
{
"guardrail_name": _KEYWORD_FILTER,
"litellm_params": {
"guardrail": "litellm_content_filter",
"mode": "post_call",
"default_on": False,
"blocked_words": [
{"keyword": "persimmon", "action": "MASK"},
{"keyword": "kumquat", "action": "BLOCK"},
],
},
},
],
"policies": {
_POLICY: {
"guardrails": {"add": [_KEYWORD_FILTER]},
"pipeline": {
"mode": "post_call",
"steps": [{"guardrail": _KEYWORD_FILTER, "on_pass": "allow", "on_fail": "block"}],
},
}
},
"policy_attachments": [{"policy": _POLICY, "models": [_GOVERNED_MODEL]}],
}
path: Final = directory / f"text-rebuild-{uuid.uuid4().hex}.yaml"
path.write_text(yaml.safe_dump(config))
@ -163,14 +203,14 @@ def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
yield _Rig(wire, owned)
def _stream(rig: _Rig, identity: str) -> _Stream:
def _stream(rig: _Rig, identity: str, model: str = _TEXT_MODEL) -> _Stream:
gateway: Final = rig.proxy.gateway
with gateway.client.stream(
"POST",
"/v1/completions",
headers={"Authorization": f"Bearer {gateway.key}"},
json={
"model": _TEXT_MODEL,
"model": model,
"prompt": identity,
"max_tokens": 16,
"stream": True,
@ -212,6 +252,37 @@ def test_guarded_text_completion_stream_reaches_the_caller_whole_with_its_usage(
assert _GUARDRAIL in list_value(object_value(row["metadata"])["applied_guardrails"]), row
def test_post_call_pipeline_masks_a_streamed_text_completion(rig: _Rig) -> None:
identity: Final = f"cmpl-mask-{uuid.uuid4().hex}"
stream: Final = _stream(rig, identity, _GOVERNED_MODEL)
assert stream.status == 200, stream.body
assert stream.body.rstrip().endswith(b"data: [DONE]"), stream.body[-400:]
assert stream.text == "A [KEYWORD_REDACTED] tart", stream.frames
assert b"persimmon" not in stream.body, stream.body
assert stream.finish_reasons == ("length",), stream.frames
assert [(usage["prompt_tokens"], usage["completion_tokens"]) for usage in stream.usages] == [
(_PROMPT_TOKENS, _COMPLETION_TOKENS)
], stream.frames
(row,) = eventually(
lambda: rig.proxy.gateway.rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (identity,)),
lambda rows: len(rows) == 1,
seconds=70,
)
assert _KEYWORD_FILTER in list_value(object_value(row["metadata"])["applied_guardrails"]), row
def test_post_call_pipeline_blocks_a_streamed_text_completion(rig: _Rig) -> None:
identity: Final = f"cmpl-block-{uuid.uuid4().hex}"
stream: Final = _stream(rig, identity, _GOVERNED_MODEL)
assert stream.status == 400, stream.body
assert b"Content blocked: keyword 'kumquat' detected" in stream.body, stream.body
assert b"quat tart" not in stream.body, stream.body
def _together_chunk(choices: list[JsonValue], **extra: JsonValue) -> bytes:
return _frame(
{

View file

@ -6,6 +6,7 @@ from typing import List, Optional, Tuple
from unittest.mock import MagicMock
import pytest
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import CustomGuardrail
@ -13,7 +14,11 @@ from litellm.llms import get_guardrail_translation_mapping
from litellm.llms.openai.completion.guardrail_translation.handler import (
OpenAITextCompletionHandler,
)
from litellm.types.utils import CallTypes, TextChoices, TextCompletionResponse
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import UnifiedLLMGuardrails
from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
from litellm.types.utils import CallTypes, TextChoices, TextCompletionResponse, Usage
class MockGuardrail(CustomGuardrail):
@ -341,3 +346,143 @@ class TestPIIMaskingScenario:
assert "[EMAIL_REDACTED]" in result["prompt"][1]
assert "alice@example.com" not in result["prompt"][0]
assert "bob@test.org" not in result["prompt"][1]
class _RecordingKeywordFilter(ContentFilterGuardrail):
def __init__(self):
super().__init__(
guardrail_name="keyword-filter",
blocked_words=[
BlockedWord(keyword="persimmon", action=ContentFilterAction.MASK),
BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK),
],
event_hook=GuardrailEventHooks.post_call,
default_on=True,
)
self.streaming_sampling_rate = 1
self.scanned_texts: tuple[tuple[str, ...], ...] = ()
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
self.scanned_texts = (*self.scanned_texts, tuple(inputs.get("texts") or ()))
return await super().apply_guardrail(inputs, request_data, input_type, logging_obj)
def _text_chunk(index: int, text: str | None, finish_reason: str | None = None) -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-stream",
created=1589478378,
model="gpt-3.5-turbo-instruct",
object="text_completion",
choices=[TextChoices(index=index, text=text, finish_reason=finish_reason)],
)
def _usage_chunk() -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-stream",
created=1589478378,
model="gpt-3.5-turbo-instruct",
object="text_completion",
choices=[],
usage=Usage(prompt_tokens=3, completion_tokens=5, total_tokens=8),
)
def _delivered(chunks: list[TextCompletionResponse]) -> list[tuple[int, str | None, str | None]]:
return [(choice.index, choice.text, choice.finish_reason) for chunk in chunks for choice in chunk.choices]
async def _drive_flat_stream(guardrail: CustomGuardrail, chunks: list[TextCompletionResponse]) -> list[object]:
async def upstream():
for chunk in chunks:
yield chunk
return [
item
async for item in UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook(
user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/completions"),
response=upstream(),
request_data={"guardrail_to_apply": guardrail, "model": "gpt-3.5-turbo-instruct"},
)
]
class TestStreamingOutputProcessing:
@pytest.mark.asyncio
async def test_ended_stream_delivers_each_choice_masked_text(self):
guardrail = _RecordingKeywordFilter()
chunks = [
_text_chunk(0, "A pers"),
_text_chunk(1, "kiwi "),
_text_chunk(0, "im"),
_text_chunk(1, "pie", "stop"),
_text_chunk(0, "mon tart"),
_text_chunk(0, None, "length"),
_usage_chunk(),
]
await OpenAITextCompletionHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=guardrail,
request_data={},
deliver_ended_stream_rewrites=True,
)
assert guardrail.scanned_texts == (("A persimmon tart", "kiwi pie"),)
assert _delivered(chunks) == [
(0, "A [KEYWORD_REDACTED] tart", None),
(1, "kiwi ", None),
(0, "", None),
(1, "pie", "stop"),
(0, "", None),
(0, None, "length"),
]
assert chunks[-1].usage.total_tokens == 8
@pytest.mark.asyncio
async def test_ended_stream_scanned_without_delivery_keeps_chunks_and_stores_the_assembled_response(self):
chunks = [_text_chunk(0, "A pers"), _text_chunk(0, "immon tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
request_data: dict[str, object] = {}
await OpenAITextCompletionHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=_RecordingKeywordFilter(), request_data=request_data
)
assert _delivered(chunks) == [(0, "A pers", None), (0, "immon tart", None), (0, "", "stop")]
assembled = request_data["response"]
assert isinstance(assembled, TextCompletionResponse)
assert [(c.text, c.finish_reason) for c in assembled.choices] == [("A [KEYWORD_REDACTED] tart", "stop")]
assert assembled.usage.total_tokens == 8
@pytest.mark.asyncio
async def test_ended_stream_block_raises_the_guardrail_error(self):
chunks = [_text_chunk(0, "A kum"), _text_chunk(0, "quat tart"), _text_chunk(0, "", "stop")]
with pytest.raises(HTTPException) as exc_info:
await OpenAITextCompletionHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=_RecordingKeywordFilter(), deliver_ended_stream_rewrites=True
)
assert exc_info.value.status_code == 400
assert "kumquat" in str(exc_info.value.detail)
@pytest.mark.asyncio
async def test_flat_stream_is_scanned_once_after_it_ends(self):
guardrail = _RecordingKeywordFilter()
chunks = [_text_chunk(0, "A pers"), _text_chunk(0, "immon tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
delivered = await _drive_flat_stream(guardrail, chunks)
assert guardrail.scanned_texts == (("A persimmon tart",),)
assert len(delivered) == len(chunks)
@pytest.mark.asyncio
async def test_flat_stream_block_ends_with_an_in_band_error_frame(self):
chunks = [_text_chunk(0, "A kum"), _text_chunk(0, "quat tart"), _text_chunk(0, "", "stop"), _usage_chunk()]
delivered = await _drive_flat_stream(_RecordingKeywordFilter(), chunks)
error_frame = delivered[-1]
assert isinstance(error_frame, bytes)
assert error_frame.startswith(b'data: {"error": ')
assert b"kumquat" in error_frame

View file

@ -5,6 +5,7 @@ import pytest
from fastapi import HTTPException
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.llms.openai.completion.guardrail_translation.handler import OpenAITextCompletionHandler
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.guardrails.guardrail_hooks.grayswan import grayswan as grayswan_module
from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import (
@ -12,6 +13,7 @@ from litellm.proxy.guardrails.guardrail_hooks.grayswan.grayswan import (
GraySwanGuardrailAPIError,
)
from litellm.types.guardrails import GuardrailEventHooks
from litellm.types.utils import TextChoices, TextCompletionResponse
@pytest.fixture
@ -884,3 +886,24 @@ async def test_pre_call_payload_unchanged() -> None:
{"role": "user", "content": "second"},
]
assert "tools" not in payload
@pytest.mark.asyncio
async def test_post_call_on_a_text_completion_forwards_only_the_key_and_team_metadata() -> None:
guardrail = _post_call_guardrail()
client = _CapturingClient()
guardrail.async_handler = client
await OpenAITextCompletionHandler().process_output_response(
response=TextCompletionResponse(
id="cmpl-1", model="gpt-4.1-mini", choices=[TextChoices(index=0, text="hello", finish_reason="stop")]
),
guardrail_to_apply=guardrail,
user_api_key_dict=UserAPIKeyAuth(team_id="team-456", key_alias="ci-key"),
request_data={"model": "gpt-4.1-mini", "prompt": "say hello", "metadata": {"requester_note": "private"}},
)
forwarded = client.calls[0]["json"]["litellm_metadata"]
assert forwarded["user_api_key_team_id"] == "team-456"
assert forwarded["user_api_key_key_alias"] == "ci-key"
assert "requester_note" not in forwarded

View file

@ -41,6 +41,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
GuardrailPipeline,
PipelineStep,
)
from litellm.types.utils import TextChoices, TextCompletionResponse
@pytest.fixture(autouse=True)
@ -1886,7 +1887,7 @@ def test_streamable_post_call_pipelines_keeps_hook_guardrails_and_drops_iterator
@pytest.mark.parametrize(
"request_route",
["/v1/completions", "/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"],
["/v1beta/models/gemini-2.5-flash:streamGenerateContent", "/a2a/agent"],
)
def test_streamable_post_call_pipelines_keeps_legacy_hooks_off_routes_that_assemble_no_response(
make_user_api_key_auth, monkeypatch, caplog, request_route
@ -2036,6 +2037,120 @@ async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_every_cho
assert _warnings(caplog) == []
def _text_completion_chunk(text: str, finish_reason: str | None = None) -> TextCompletionResponse:
return TextCompletionResponse(
id="cmpl-1", model="m", choices=[TextChoices(index=0, text=text, finish_reason=finish_reason)]
)
def _rewritten_text_completion(response: Any) -> TextCompletionResponse:
return TextCompletionResponse(
id=response.id,
model=response.model,
choices=[TextChoices(index=0, text="[REWRITTEN] " + response.choices[0].text, finish_reason="stop")],
)
@pytest.mark.asyncio
async def test_streaming_iterator_hook_delivers_legacy_hook_rewrite_on_a_text_completion_stream(
proxy_logging, make_user_api_key_auth, monkeypatch, caplog
):
seen: Dict[str, Any] = {}
monkeypatch.setattr(litellm, "callbacks", [_legacy_hook_stream_guardrail(seen, rewrite=_rewritten_text_completion)])
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
data = {**_post_call_pipeline_data(stream=True), "prompt": "hi"}
chunks = [_text_completion_chunk("hello "), _text_completion_chunk("world"), _text_completion_chunk("", "stop")]
auth = make_user_api_key_auth(request_route="/v1/completions")
with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
await proxy_logging.pre_call_hook(
user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True
)
delivered = [
item
async for item in proxy_logging.async_post_call_streaming_iterator_hook(
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
)
]
assert isinstance(seen["response"], TextCompletionResponse)
assert seen["response"].choices[0].text == "hello world"
assert [(item.choices[0].text, item.choices[0].finish_reason) for item in delivered] == [
("[REWRITTEN] hello world", None),
("", None),
("", "stop"),
]
assert data["metadata"]["applied_guardrails"] == ["gr-post"]
assert _warnings(caplog) == []
class _FlatBlockGuardrail(CustomGuardrail):
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
if any("kumquat" in text for text in inputs.get("texts") or []):
raise HTTPException(status_code=400, detail="flat guardrail blocked kumquat")
return inputs
@pytest.mark.asyncio
async def test_streaming_iterator_hook_delivers_a_flat_guardrail_block_on_a_text_completion_stream_under_a_pipeline(
proxy_logging, make_user_api_key_auth, monkeypatch
):
seen: Final[Dict[str, Any]] = {}
flat: Final = _FlatBlockGuardrail(
guardrail_name="flat-block", event_hook=GuardrailEventHooks.post_call, default_on=True
)
monkeypatch.setattr(litellm, "callbacks", [flat, _live_scan_guardrail(seen)])
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
data: Final = {**_post_call_pipeline_data(stream=True), "prompt": "hi"}
chunks: Final = [
_text_completion_chunk("A kum"),
_text_completion_chunk("quat tart"),
_text_completion_chunk("", "stop"),
]
auth: Final = make_user_api_key_auth(request_route="/v1/completions")
await proxy_logging.pre_call_hook(user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True)
delivered: Final = [
item
async for item in proxy_logging.async_post_call_streaming_iterator_hook(
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
)
]
assert seen["texts"] == ["A kumquat tart"]
assert isinstance(delivered[-1], bytes)
assert json.loads(delivered[-1].decode().removeprefix("data: "))["error"]["message"] == (
"flat guardrail blocked kumquat"
)
@pytest.mark.asyncio
async def test_streaming_iterator_hook_live_pipeline_scans_what_a_disconnected_text_completion_client_received(
proxy_logging, make_user_api_key_auth, monkeypatch
):
seen: Final[Dict[str, Any]] = {}
monkeypatch.setattr(litellm, "callbacks", [_live_scan_guardrail(seen)])
monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
data: Final = {**_post_call_pipeline_data(step=_DETECT_ONLY_STEP, stream=True), "prompt": "hi"}
chunks: Final = [
_text_completion_chunk("hello "),
_text_completion_chunk("world"),
_text_completion_chunk("", "stop"),
]
auth: Final = make_user_api_key_auth(request_route="/v1/completions")
await proxy_logging.pre_call_hook(user_api_key_dict=auth, data=data, call_type="text_completion", guardrails_only=True)
stream: Final = proxy_logging.async_post_call_streaming_iterator_hook(
user_api_key_dict=auth, response=_async_chunk_iter(chunks), request_data=data
)
first: Final = await stream.__anext__()
await stream.aclose()
assert first.choices[0].text == "hello "
assert seen["texts"] == ["hello "]
assert _post_call_statuses(data) == ["success"]
@pytest.mark.asyncio
async def test_streaming_iterator_hook_releases_stream_untouched_when_legacy_hook_returns_none(
proxy_logging, make_user_api_key_auth, monkeypatch