fix(guardrails): inspect every choice before raising an undeliverable rewrite

This commit is contained in:
Alessandro Pignati 2026-10-28 12:02:25 +01:00
parent ae1ccac316
commit 7254b3ad5e
2 changed files with 143 additions and 21 deletions

View file

@ -669,7 +669,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if pre_guardrail_tool_calls and len(model_response.choices) > 1
else (model_response,)
)
# Local import avoids a module-level cyclic import with the policy engine.
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
inspection_request_data: Final = request_data if request_data is not None else {}
missing: Final = object()
prior_response: Final = inspection_request_data.get("response", missing)
prior_responses: Final = inspection_request_data.get("responses", missing)
# An undeliverable rewrite on one choice must not end the loop: the caller releases the
# original chunks when it catches this, so leaving later choices uninspected would let a
# request with n > 1 pair a redactable choice with a prohibited one and have the whole
# response delivered unchecked. Inspect every choice, then raise the first failure.
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
try:
for inspection_response in inspection_responses:
inspection_request_data["response"] = inspection_response
@ -681,16 +692,31 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
if len(inspection_response.choices) == 1
else responses_so_far
)
await self.process_output_response(
response=inspection_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=inspection_request_data,
)
try:
await self.process_output_response(
response=inspection_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=inspection_request_data,
)
except UndeliverableStreamRewrite as exc:
if undeliverable is None:
undeliverable = exc
finally:
inspection_request_data["response"] = model_response
inspection_request_data["responses"] = responses_so_far
# Restore the caller's keys rather than overwriting them: the inspection loop owns
# these entries only for its own duration, and "responses" is not a key the
# non-streaming path sets at all.
if prior_response is missing:
inspection_request_data.pop("response", None)
else:
inspection_request_data["response"] = prior_response
if prior_responses is missing:
inspection_request_data.pop("responses", None)
else:
inspection_request_data["responses"] = prior_responses
if undeliverable is not None:
raise undeliverable
if not deliver_ended_stream_rewrites:
return
await self._write_ended_stream_text_rewrites(
@ -835,23 +861,38 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
)
for choice in assembled.choices
)
# Local import avoids a module-level cyclic import with the policy engine.
from litellm.proxy.policy_engine.pipeline_executor import (
UndeliverableStreamRewrite,
)
# Same rule as the ended-stream inspection loop: an undeliverable rewrite on one
# choice must not leave the rest uninspected, because the caller releases the
# original chunks when it catches this.
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
try:
for choice, choice_sink, choice_chunks in choice_rounds:
resolved_request_data["response"] = assembled.model_copy(
update=MappingProxyType({"choices": [choice]})
)
resolved_request_data["responses"] = choice_chunks
await self._process_streaming_transform(
responses_so_far=choice_chunks,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=resolved_request_data,
sink=choice_sink,
)
try:
await self._process_streaming_transform(
responses_so_far=choice_chunks,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=resolved_request_data,
sink=choice_sink,
)
except UndeliverableStreamRewrite as exc:
if transform_undeliverable is None:
transform_undeliverable = exc
finally:
resolved_request_data["response"] = assembled
resolved_request_data["responses"] = responses_so_far
if transform_undeliverable is not None:
raise transform_undeliverable
merged_texts: Final = MappingProxyType(
dict(
chain.from_iterable(

View file

@ -1433,10 +1433,91 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
)
assert request_data["visited_choices"] == (0, 1)
assert request_data["metadata"]["trace"] == "retained"
assert request_data["responses"] is chunks
restored: Final = request_data["response"]
assert isinstance(restored, ModelResponse)
assert tuple(choice.index for choice in restored.choices) == (0, 1)
if transform:
# The transform path seeds "response" and "responses" up front as the context the
# guardrail reads, so they are its own values rather than leftovers.
assert request_data["responses"] is chunks
restored: Final = request_data["response"]
assert isinstance(restored, ModelResponse)
assert tuple(choice.index for choice in restored.choices) == (0, 1)
else:
# The ended-stream inspection loop owns these keys only while it runs; neither was in
# request_data beforehand, and the non-streaming path never sets "responses" at all.
assert "response" not in request_data
assert "responses" not in request_data
@pytest.mark.asyncio
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
"""An undeliverable rewrite on one choice must not leave later choices uninspected.
The caller releases the original chunks when it catches UndeliverableStreamRewrite, so
bailing out of the loop would let a request with n > 1 pair a redactable choice with a
prohibited one and have the whole response delivered unchecked.
"""
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None:
"""The transform path runs its own per-choice loop and needs the same rule."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
stream_transform_sink=StreamTransformSink(),
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: