diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..7b1eaca13af 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1414,6 +1414,9 @@ class CustomGuardrail(CustomLogger): end_time=end_time, duration=duration, masked_entity_count=masked_entity_count, + # Set by callers that inspect one choice at a time, so the per-choice entries of an + # n > 1 response can be told apart. + choice_index=request_data.get("guardrail_choice_index"), **(tracing_detail or {}), ) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 5b8b31a406f..705c98564cd 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -669,10 +669,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if pre_guardrail_tool_calls and len(model_response.choices) > 1 else (model_response,) ) + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + inspection_request_data: Final = request_data if request_data is not None else {} + missing: Final = object() + prior_response: Final = inspection_request_data.get("response", missing) + prior_responses: Final = inspection_request_data.get("responses", missing) + prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) + undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response + inspection_request_data["guardrail_choice_index"] = ( + inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None + ) inspection_request_data["responses"] = ( [ self._narrowed_to_choice(chunk, inspection_response.choices[0].index) @@ -681,16 +691,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if len(inspection_response.choices) == 1 else responses_so_far ) - await self.process_output_response( - response=inspection_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=inspection_request_data, - ) + try: + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + except UndeliverableStreamRewrite as exc: + if undeliverable is None: + undeliverable = exc finally: - inspection_request_data["response"] = model_response - inspection_request_data["responses"] = responses_so_far + if prior_response is missing: + inspection_request_data.pop("response", None) + else: + inspection_request_data["response"] = prior_response + if prior_responses is missing: + inspection_request_data.pop("responses", None) + else: + inspection_request_data["responses"] = prior_responses + if prior_choice_index is missing: + inspection_request_data.pop("guardrail_choice_index", None) + else: + inspection_request_data["guardrail_choice_index"] = prior_choice_index + if undeliverable is not None: + raise undeliverable if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -705,6 +731,50 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + @staticmethod + def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]: + for chunk in responses_so_far: + for choice in chunk.choices: + yield choice.index + + @staticmethod + def _streamed_argument_fragments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> Iterator[tuple[tuple[int, int], str]]: + for chunk in responses_so_far: + for choice in chunk.choices: + for call in choice.delta.tool_calls or (): + arguments = getattr(getattr(call, "function", None), "arguments", None) + yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else "" + + @staticmethod + def _parses_as_json(arguments: str) -> bool: + if not arguments: + return False + try: + json.loads(arguments) + except ValueError: + return False + return True + + @staticmethod + def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: + """Whether every streamed tool call now carries arguments that parse as JSON. + + A call that has announced itself but emitted no argument text yet is still in flight, so a + parallel call finishing first does not make the whole set complete. + """ + fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) + if not fragments: + return False + joined: Final = MappingProxyType( + { + call: "".join(text for other, text in fragments if other == call) + for call in frozenset(call for call, _ in fragments) + } + ) + return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) + @staticmethod def _rebuild_ended_stream_per_choice( responses_so_far: Sequence["ModelResponseStream"], @@ -824,9 +894,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - resolved_request_data["response"] = assembled - if len(assembled.choices) > 1: + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) + tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) + assembled: Final = ( + None + if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete + else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + ) + if assembled is not None: + resolved_request_data["response"] = assembled + if assembled is not None and len(assembled.choices) > 1: choice_rounds: Final = tuple( ( choice, @@ -835,23 +913,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for choice in assembled.choices ) + from litellm.proxy.policy_engine.pipeline_executor import ( + UndeliverableStreamRewrite, + ) + + transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: resolved_request_data["response"] = assembled.model_copy( update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks - await self._process_streaming_transform( - responses_so_far=choice_chunks, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=resolved_request_data, - sink=choice_sink, - ) + resolved_request_data["guardrail_choice_index"] = choice.index + try: + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + except UndeliverableStreamRewrite as exc: + if transform_undeliverable is None: + transform_undeliverable = exc finally: resolved_request_data["response"] = assembled resolved_request_data["responses"] = responses_so_far + resolved_request_data.pop("guardrail_choice_index", None) + if transform_undeliverable is not None: + raise transform_undeliverable merged_texts: Final = MappingProxyType( dict( chain.from_iterable( @@ -868,14 +959,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) sink.record(merged_texts, merged_holdback) return - tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) - inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( - tuple( - MappingProxyType({"index": index, **converted}) - for index, tool_call in enumerate(tool_calls) - if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + if assembled is not None and tool_arguments_complete: + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) ) - ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..d27c4bb62a6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3511,6 +3511,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset( "end_time", "duration", "masked_entity_count", + "choice_index", "guardrail_id", "policy_template", "detection_method", @@ -3551,6 +3552,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): } """ + choice_index: ReadOnly[int | None] + """Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice.""" + guardrail_id: str | None """Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'""" diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index b3649c9daaa..04b2b036bea 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1437,10 +1437,247 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" - assert request_data["responses"] is chunks - restored: Final = request_data["response"] - assert isinstance(restored, ModelResponse) - assert tuple(choice.index for choice in restored.choices) == (0, 1) + if transform: + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) + else: + assert "response" not in request_data + assert "responses" not in request_data + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None: + """An undeliverable rewrite on one choice must not leave later choices uninspected. + + The caller releases the original chunks when it catches UndeliverableStreamRewrite, so + bailing out of the loop would let a request with n > 1 pair a redactable choice with a + prohibited one and have the whole response delivered unchecked. + """ + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + ) + assert request_data["visited_choices"] == (0, 1) + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None: + """The transform path runs its own per-choice loop and needs the same rule.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink(), + ) + assert request_data["visited_choices"] == (0, 1) + + @pytest.mark.asyncio + async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None: + """A buffered stream has no finish_reason yet, so the ended path must key off the opt-in. + + Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered + caller on the in-flight path, where rewrites are never written back. + """ + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class PathProbe(CustomGuardrail): + saw_assembled_response: bool = False + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse) + return inputs + + chunks: Final = [ + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)] + ) + ] + assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=PathProbe(guardrail_name="probe"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + assert PathProbe.saw_assembled_response is True + + @pytest.mark.asyncio + async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None: + """The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks. + + Reporting an empty holdback here would release text the guardrail asked to withhold. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class HoldingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + texts: Final = inputs.get("texts") or () + return {**inputs, "stream_holdback_chars": [len(text) for text in texts]} + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("alpha", "bravo-long"))])] + sink: Final = StreamTransformSink() + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"), + litellm_logging_obj=None, + stream_transform_sink=sink, + ) + assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")} + + @pytest.mark.asyncio + async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None: + """Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent. + + Sending them would hand the guardrail something like `{"ci`, which main never did. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(inputs.get("tool_calls")) + return inputs + + def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + }]), finish_reason=finish_reason, + )]) + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + partial: Final = [chunk("looking", '{"ci', None)] + await handler.process_output_streaming_response( + responses_so_far=partial, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")] + await handler.process_output_streaming_response( + responses_so_far=complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + + @pytest.mark.asyncio + async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None: + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(inputs.get("tool_calls")) + return inputs + + def call(index: int, arguments: str) -> dict[str, object]: + return { + "index": index, "id": f"call_{index}", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + } + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]), + finish_reason=None, + )])] + await handler.process_output_streaming_response( + responses_so_far=one_still_empty, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + both_complete: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, + delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]), + finish_reason="tool_calls", + )])] + await handler.process_output_streaming_response( + responses_so_far=both_complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: