Merge remote-tracking branch 'origin/litellm_guardrail_choice_isolation' into litellm_guardrail_stream_inspection

This commit is contained in:
Alessandro Pignati 2026-10-29 12:56:05 +01:00
commit 5de1fcda40
4 changed files with 367 additions and 31 deletions

View file

@ -1414,6 +1414,9 @@ class CustomGuardrail(CustomLogger):
end_time=end_time,
duration=duration,
masked_entity_count=masked_entity_count,
# Set by callers that inspect one choice at a time, so the per-choice entries of an
# n > 1 response can be told apart.
choice_index=request_data.get("guardrail_choice_index"),
**(tracing_detail or {}),
)

View file

@ -669,10 +669,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if pre_guardrail_tool_calls and len(model_response.choices) > 1
else (model_response,)
)
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
inspection_request_data: Final = request_data if request_data is not None else {}
missing: Final = object()
prior_response: Final = inspection_request_data.get("response", missing)
prior_responses: Final = inspection_request_data.get("responses", missing)
prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing)
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
try:
for inspection_response in inspection_responses:
inspection_request_data["response"] = inspection_response
inspection_request_data["guardrail_choice_index"] = (
inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None
)
inspection_request_data["responses"] = (
[
self._narrowed_to_choice(chunk, inspection_response.choices[0].index)
@ -681,16 +691,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if len(inspection_response.choices) == 1
else responses_so_far
)
await self.process_output_response(
response=inspection_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=inspection_request_data,
)
try:
await self.process_output_response(
response=inspection_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=inspection_request_data,
)
except UndeliverableStreamRewrite as exc:
if undeliverable is None:
undeliverable = exc
finally:
inspection_request_data["response"] = model_response
inspection_request_data["responses"] = responses_so_far
if prior_response is missing:
inspection_request_data.pop("response", None)
else:
inspection_request_data["response"] = prior_response
if prior_responses is missing:
inspection_request_data.pop("responses", None)
else:
inspection_request_data["responses"] = prior_responses
if prior_choice_index is missing:
inspection_request_data.pop("guardrail_choice_index", None)
else:
inspection_request_data["guardrail_choice_index"] = prior_choice_index
if undeliverable is not None:
raise undeliverable
if not deliver_ended_stream_rewrites:
return
await self._write_ended_stream_text_rewrites(
@ -705,6 +731,50 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@staticmethod
def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]:
for chunk in responses_so_far:
for choice in chunk.choices:
yield choice.index
@staticmethod
def _streamed_argument_fragments(
responses_so_far: Sequence["ModelResponseStream"],
) -> Iterator[tuple[tuple[int, int], str]]:
for chunk in responses_so_far:
for choice in chunk.choices:
for call in choice.delta.tool_calls or ():
arguments = getattr(getattr(call, "function", None), "arguments", None)
yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else ""
@staticmethod
def _parses_as_json(arguments: str) -> bool:
if not arguments:
return False
try:
json.loads(arguments)
except ValueError:
return False
return True
@staticmethod
def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool:
"""Whether every streamed tool call now carries arguments that parse as JSON.
A call that has announced itself but emitted no argument text yet is still in flight, so a
parallel call finishing first does not make the whole set complete.
"""
fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far))
if not fragments:
return False
joined: Final = MappingProxyType(
{
call: "".join(text for other, text in fragments if other == call)
for call in frozenset(call for call, _ in fragments)
}
)
return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values())
@staticmethod
def _rebuild_ended_stream_per_choice(
responses_so_far: Sequence["ModelResponseStream"],
@ -824,9 +894,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if self._streamed_tool_call_fingerprints(responses_so_far):
assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
resolved_request_data["response"] = assembled
if len(assembled.choices) > 1:
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far))
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
assembled: Final = (
None
if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
)
if assembled is not None:
resolved_request_data["response"] = assembled
if assembled is not None and len(assembled.choices) > 1:
choice_rounds: Final = tuple(
(
choice,
@ -835,23 +913,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
)
for choice in assembled.choices
)
from litellm.proxy.policy_engine.pipeline_executor import (
UndeliverableStreamRewrite,
)
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
try:
for choice, choice_sink, choice_chunks in choice_rounds:
resolved_request_data["response"] = assembled.model_copy(
update=MappingProxyType({"choices": [choice]})
)
resolved_request_data["responses"] = choice_chunks
await self._process_streaming_transform(
responses_so_far=choice_chunks,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=resolved_request_data,
sink=choice_sink,
)
resolved_request_data["guardrail_choice_index"] = choice.index
try:
await self._process_streaming_transform(
responses_so_far=choice_chunks,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=resolved_request_data,
sink=choice_sink,
)
except UndeliverableStreamRewrite as exc:
if transform_undeliverable is None:
transform_undeliverable = exc
finally:
resolved_request_data["response"] = assembled
resolved_request_data["responses"] = responses_so_far
resolved_request_data.pop("guardrail_choice_index", None)
if transform_undeliverable is not None:
raise transform_undeliverable
merged_texts: Final = MappingProxyType(
dict(
chain.from_iterable(
@ -868,14 +959,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
)
sink.record(merged_texts, merged_holdback)
return
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
tuple(
MappingProxyType({"index": index, **converted})
for index, tool_call in enumerate(tool_calls)
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
if assembled is not None and tool_arguments_complete:
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
tuple(
MappingProxyType({"index": index, **converted})
for index, tool_call in enumerate(tool_calls)
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
)
)
)
if responses_so_far and getattr(responses_so_far[0], "model", None):
inputs["model"] = responses_so_far[0].model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(

View file

@ -3511,6 +3511,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
"end_time",
"duration",
"masked_entity_count",
"choice_index",
"guardrail_id",
"policy_template",
"detection_method",
@ -3551,6 +3552,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
}
"""
choice_index: ReadOnly[int | None]
"""Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice."""
guardrail_id: str | None
"""Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'"""

View file

@ -1437,10 +1437,247 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
)
assert request_data["visited_choices"] == (0, 1)
assert request_data["metadata"]["trace"] == "retained"
assert request_data["responses"] is chunks
restored: Final = request_data["response"]
assert isinstance(restored, ModelResponse)
assert tuple(choice.index for choice in restored.choices) == (0, 1)
if transform:
assert request_data["responses"] is chunks
restored: Final = request_data["response"]
assert isinstance(restored, ModelResponse)
assert tuple(choice.index for choice in restored.choices) == (0, 1)
else:
assert "response" not in request_data
assert "responses" not in request_data
@pytest.mark.asyncio
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
"""An undeliverable rewrite on one choice must not leave later choices uninspected.
The caller releases the original chunks when it catches UndeliverableStreamRewrite, so
bailing out of the loop would let a request with n > 1 pair a redactable choice with a
prohibited one and have the whole response delivered unchecked.
"""
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None:
"""The transform path runs its own per-choice loop and needs the same rule."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
stream_transform_sink=StreamTransformSink(),
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None:
"""A buffered stream has no finish_reason yet, so the ended path must key off the opt-in.
Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered
caller on the in-flight path, where rewrites are never written back.
"""
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class PathProbe(CustomGuardrail):
saw_assembled_response: bool = False
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse)
return inputs
chunks: Final = [
ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)]
)
]
assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=PathProbe(guardrail_name="probe"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert PathProbe.saw_assembled_response is True
@pytest.mark.asyncio
async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None:
"""The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks.
Reporting an empty holdback here would release text the guardrail asked to withhold.
"""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class HoldingGuardrail(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
texts: Final = inputs.get("texts") or ()
return {**inputs, "stream_holdback_chars": [len(text) for text in texts]}
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("alpha", "bravo-long"))])]
sink: Final = StreamTransformSink()
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"),
litellm_logging_obj=None,
stream_transform_sink=sink,
)
assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")}
@pytest.mark.asyncio
async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None:
"""Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent.
Sending them would hand the guardrail something like `{"ci`, which main never did.
"""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(inputs.get("tool_calls"))
return inputs
def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream:
return ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": "call_0", "type": "function",
"function": {"name": "weather", "arguments": arguments},
}]), finish_reason=finish_reason,
)])
RecordingGuardrail.seen = []
handler: Final = OpenAIChatCompletionsHandler()
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
partial: Final = [chunk("looking", '{"ci', None)]
await handler.process_output_streaming_response(
responses_so_far=partial, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert RecordingGuardrail.seen == [None]
RecordingGuardrail.seen = []
complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")]
await handler.process_output_streaming_response(
responses_so_far=complete, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
sent: Final = RecordingGuardrail.seen[-1]
assert sent is not None
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
@pytest.mark.asyncio
async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None:
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(inputs.get("tool_calls"))
return inputs
def call(index: int, arguments: str) -> dict[str, object]:
return {
"index": index, "id": f"call_{index}", "type": "function",
"function": {"name": "weather", "arguments": arguments},
}
RecordingGuardrail.seen = []
handler: Final = OpenAIChatCompletionsHandler()
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
finish_reason=None,
)])]
await handler.process_output_streaming_response(
responses_so_far=one_still_empty, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert RecordingGuardrail.seen == [None]
RecordingGuardrail.seen = []
both_complete: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0,
delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]),
finish_reason="tool_calls",
)])]
await handler.process_output_streaming_response(
responses_so_far=both_complete, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
sent: Final = RecordingGuardrail.seen[-1]
assert sent is not None
assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}']
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: