From 7254b3ad5ee17b6270a474ee2f8b84833be4cbfd Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:02:25 +0100 Subject: [PATCH 1/7] fix(guardrails): inspect every choice before raising an undeliverable rewrite --- .../chat/guardrail_translation/handler.py | 75 ++++++++++++---- .../test_openai_guardrail_handler.py | 89 ++++++++++++++++++- 2 files changed, 143 insertions(+), 21 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a4a3956df94..9ad42b18c70 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -669,7 +669,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if pre_guardrail_tool_calls and len(model_response.choices) > 1 else (model_response,) ) + # Local import avoids a module-level cyclic import with the policy engine. + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + inspection_request_data: Final = request_data if request_data is not None else {} + missing: Final = object() + prior_response: Final = inspection_request_data.get("response", missing) + prior_responses: Final = inspection_request_data.get("responses", missing) + # An undeliverable rewrite on one choice must not end the loop: the caller releases the + # original chunks when it catches this, so leaving later choices uninspected would let a + # request with n > 1 pair a redactable choice with a prohibited one and have the whole + # response delivered unchecked. Inspect every choice, then raise the first failure. + undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response @@ -681,16 +692,31 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if len(inspection_response.choices) == 1 else responses_so_far ) - await self.process_output_response( - response=inspection_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=inspection_request_data, - ) + try: + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + except UndeliverableStreamRewrite as exc: + if undeliverable is None: + undeliverable = exc finally: - inspection_request_data["response"] = model_response - inspection_request_data["responses"] = responses_so_far + # Restore the caller's keys rather than overwriting them: the inspection loop owns + # these entries only for its own duration, and "responses" is not a key the + # non-streaming path sets at all. + if prior_response is missing: + inspection_request_data.pop("response", None) + else: + inspection_request_data["response"] = prior_response + if prior_responses is missing: + inspection_request_data.pop("responses", None) + else: + inspection_request_data["responses"] = prior_responses + if undeliverable is not None: + raise undeliverable if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -835,23 +861,38 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for choice in assembled.choices ) + # Local import avoids a module-level cyclic import with the policy engine. + from litellm.proxy.policy_engine.pipeline_executor import ( + UndeliverableStreamRewrite, + ) + + # Same rule as the ended-stream inspection loop: an undeliverable rewrite on one + # choice must not leave the rest uninspected, because the caller releases the + # original chunks when it catches this. + transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: resolved_request_data["response"] = assembled.model_copy( update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks - await self._process_streaming_transform( - responses_so_far=choice_chunks, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=resolved_request_data, - sink=choice_sink, - ) + try: + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + except UndeliverableStreamRewrite as exc: + if transform_undeliverable is None: + transform_undeliverable = exc finally: resolved_request_data["response"] = assembled resolved_request_data["responses"] = responses_so_far + if transform_undeliverable is not None: + raise transform_undeliverable merged_texts: Final = MappingProxyType( dict( chain.from_iterable( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bffe6e4c123..f21e22545f5 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1433,10 +1433,91 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" - assert request_data["responses"] is chunks - restored: Final = request_data["response"] - assert isinstance(restored, ModelResponse) - assert tuple(choice.index for choice in restored.choices) == (0, 1) + if transform: + # The transform path seeds "response" and "responses" up front as the context the + # guardrail reads, so they are its own values rather than leftovers. + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) + else: + # The ended-stream inspection loop owns these keys only while it runs; neither was in + # request_data beforehand, and the non-streaming path never sets "responses" at all. + assert "response" not in request_data + assert "responses" not in request_data + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None: + """An undeliverable rewrite on one choice must not leave later choices uninspected. + + The caller releases the original chunks when it catches UndeliverableStreamRewrite, so + bailing out of the loop would let a request with n > 1 pair a redactable choice with a + prohibited one and have the whole response delivered unchecked. + """ + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + ) + assert request_data["visited_choices"] == (0, 1) + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None: + """The transform path runs its own per-choice loop and needs the same rule.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink(), + ) + assert request_data["visited_choices"] == (0, 1) @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: From c6e386cefd25122d2eb134c3fe8a0169856a1f9e Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:13:25 +0100 Subject: [PATCH 2/7] test(guardrails): cover the buffered ended-stream path and per-choice holdback --- .../test_openai_guardrail_handler.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index f21e22545f5..bc8ffe07b4f 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1519,6 +1519,73 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) + @pytest.mark.asyncio + async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None: + """A buffered stream has no finish_reason yet, so the ended path must key off the opt-in. + + Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered + caller on the in-flight path, where rewrites are never written back. + """ + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class PathProbe(CustomGuardrail): + saw_assembled_response: bool = False + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + # Only the ended-stream inspection loop hands the guardrail an assembled + # "response"; the in-flight branch passes raw chunks under "responses" alone. + type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse) + return inputs + + chunks: Final = [ + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)] + ) + ] + assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=PathProbe(guardrail_name="probe"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + assert PathProbe.saw_assembled_response is True + + @pytest.mark.asyncio + async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None: + """The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks. + + Reporting an empty holdback here would release text the guardrail asked to withhold. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class HoldingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + texts: Final = inputs.get("texts") or () + return {**inputs, "stream_holdback_chars": [len(text) for text in texts]} + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("alpha", "bravo-long"))])] + sink: Final = StreamTransformSink() + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"), + litellm_logging_obj=None, + stream_transform_sink=sink, + ) + assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")} + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From fd3ad67c474b162200e757a0fcdb01926035e4c4 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:22:00 +0100 Subject: [PATCH 3/7] fix(guardrails): send tool calls to the guardrail only once their arguments are complete --- .../chat/guardrail_translation/handler.py | 19 ++++--- .../test_openai_guardrail_handler.py | 49 +++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 9ad42b18c70..ef9c1eb34ab 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -909,14 +909,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) sink.record(merged_texts, merged_holdback) return - tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) - inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( - tuple( - MappingProxyType({"index": index, **converted}) - for index, tool_call in enumerate(tool_calls) - if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which + # main never handed to a guardrail: one that parses them either errors or judges + # arguments that do not exist yet. Send tool calls only once the stream has finished and + # the arguments are complete; the text of those rounds is still inspected above. + if self._first_choice_has_finished(responses_so_far): + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) ) - ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bc8ffe07b4f..a503bbf386e 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1586,6 +1586,55 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")} + @pytest.mark.asyncio + async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None: + """Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent. + + Sending them would hand the guardrail something like `{"ci`, which main never did. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(inputs.get("tool_calls")) + return inputs + + def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream: + # The text keeps the guardrail in play for the partial round; without any text the + # transform path has nothing to inspect and never calls it at all. + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + }]), finish_reason=finish_reason, + )]) + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + partial: Final = [chunk("looking", '{"ci', None)] + await handler.process_output_streaming_response( + responses_so_far=partial, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")] + await handler.process_output_streaming_response( + responses_so_far=complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From ff5644b3d873bfce95e6ff300432631d135a2005 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:32:06 +0100 Subject: [PATCH 4/7] perf(guardrails): rebuild the stream only when a round consumes it --- .../chat/guardrail_translation/handler.py | 22 +++++++-- .../test_openai_guardrail_handler.py | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index ef9c1eb34ab..a43c6b193b6 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -850,9 +850,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - resolved_request_data["response"] = assembled - if len(assembled.choices) > 1: + # Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is + # quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild. + # main rebuilds only on the ended path, so rebuild here only when this round consumes the + # assembled response -- to isolate choices, or to hand over complete tool calls once the + # stream has finished. A single-choice round in flight needs neither. + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + streamed_choice_indices: Final = frozenset( + choice.index for chunk in responses_so_far for choice in chunk.choices + ) + assembled: Final = ( + None + if len(streamed_choice_indices) == 1 and not stream_ended + else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + ) + if assembled is not None: + resolved_request_data["response"] = assembled + if assembled is not None and len(assembled.choices) > 1: choice_rounds: Final = tuple( ( choice, @@ -913,7 +927,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # main never handed to a guardrail: one that parses them either errors or judges # arguments that do not exist yet. Send tool calls only once the stream has finished and # the arguments are complete; the text of those rounds is still inspected above. - if self._first_choice_has_finished(responses_so_far): + if assembled is not None and stream_ended: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( tuple( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index a503bbf386e..137383c3483 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1635,6 +1635,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + @pytest.mark.asyncio + async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None: + """Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length. + + main rebuilds only on the ended path; an in-flight single-choice round consumes nothing + from the assembled response, so it must not pay for one. + """ + from unittest.mock import patch + + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(text: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": '{"c'}, + }]), finish_reason=finish_reason, + )]) + + handler: Final = OpenAIChatCompletionsHandler() + rebuilds: Final[list[int]] = [] + original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice + + def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy + rebuilds.append(len(responses_so_far)) + return original(responses_so_far, litellm_logging_obj) + + # The real attribute is a staticmethod, so the spy has to be one too: a plain function + # would be bound and receive the handler as its first argument. + with patch.object( + OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild) + ): + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None)], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [] + + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [2] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From f2342b832d051a4800c03e737981bb932bd8d2f3 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 13:09:43 +0100 Subject: [PATCH 5/7] feat(guardrails): label per-choice log entries and gate tool calls on complete arguments --- litellm/integrations/custom_guardrail.py | 3 ++ .../chat/guardrail_translation/handler.py | 47 +++++++++++++++++-- litellm/types/utils.py | 3 ++ .../test_openai_guardrail_handler.py | 30 ++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..7b1eaca13af 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1414,6 +1414,9 @@ class CustomGuardrail(CustomLogger): end_time=end_time, duration=duration, masked_entity_count=masked_entity_count, + # Set by callers that inspect one choice at a time, so the per-choice entries of an + # n > 1 response can be told apart. + choice_index=request_data.get("guardrail_choice_index"), **(tracing_detail or {}), ) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a43c6b193b6..be698093fec 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -676,6 +676,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): missing: Final = object() prior_response: Final = inspection_request_data.get("response", missing) prior_responses: Final = inspection_request_data.get("responses", missing) + prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) # An undeliverable rewrite on one choice must not end the loop: the caller releases the # original chunks when it catches this, so leaving later choices uninspected would let a # request with n > 1 pair a redactable choice with a prohibited one and have the whole @@ -684,6 +685,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response + # Labels this round's guardrail log entry and span with the choice it inspected. + inspection_request_data["guardrail_choice_index"] = ( + inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None + ) inspection_request_data["responses"] = ( [ self._narrowed_to_choice(chunk, inspection_response.choices[0].index) @@ -715,6 +720,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inspection_request_data.pop("responses", None) else: inspection_request_data["responses"] = prior_responses + if prior_choice_index is missing: + inspection_request_data.pop("guardrail_choice_index", None) + else: + inspection_request_data["guardrail_choice_index"] = prior_choice_index if undeliverable is not None: raise undeliverable if not deliver_ended_stream_rewrites: @@ -731,6 +740,31 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + @staticmethod + def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: + """Whether every streamed tool call now carries arguments that parse as JSON. + + Streamed argument fragments are only valid JSON once the last one has arrived, so this is + the completeness test: it holds back `{"ci` and lets `{"city": "Paris"}` through whether + the stream ended normally or was cut short by a disconnect. It reads the raw chunks so the + answer is available before deciding whether to rebuild. + """ + fragments: Final[dict[tuple[int, int], list[str]]] = {} # mutable-ok: local accumulator + for chunk in responses_so_far: + for choice in chunk.choices: + for call in getattr(choice.delta, "tool_calls", None) or (): + arguments = getattr(getattr(call, "function", None), "arguments", None) + if isinstance(arguments, str) and arguments: + fragments.setdefault((choice.index, getattr(call, "index", 0)), []).append(arguments) + if not fragments: + return False + for parts in fragments.values(): + try: + json.loads("".join(parts)) + except ValueError: + return False + return True + @staticmethod def _rebuild_ended_stream_per_choice( responses_so_far: Sequence["ModelResponseStream"], @@ -859,9 +893,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): streamed_choice_indices: Final = frozenset( choice.index for chunk in responses_so_far for choice in chunk.choices ) + tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) assembled: Final = ( None - if len(streamed_choice_indices) == 1 and not stream_ended + if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) if assembled is not None: @@ -890,6 +925,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks + resolved_request_data["guardrail_choice_index"] = choice.index try: await self._process_streaming_transform( responses_so_far=choice_chunks, @@ -905,6 +941,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): finally: resolved_request_data["response"] = assembled resolved_request_data["responses"] = responses_so_far + resolved_request_data.pop("guardrail_choice_index", None) if transform_undeliverable is not None: raise transform_undeliverable merged_texts: Final = MappingProxyType( @@ -925,9 +962,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which # main never handed to a guardrail: one that parses them either errors or judges - # arguments that do not exist yet. Send tool calls only once the stream has finished and - # the arguments are complete; the text of those rounds is still inspected above. - if assembled is not None and stream_ended: + # arguments that do not exist yet. Completeness is what matters, not the finish reason: + # a stream cut short by a client disconnect carries whole arguments and no finish + # reason, and that call still has to be inspected. Send the calls once every one of + # them parses; the text of the earlier rounds is still inspected above. + if assembled is not None and tool_arguments_complete: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( tuple( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..c6eeb41a9d1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3551,6 +3551,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): } """ + choice_index: ReadOnly[int | None] + """Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice.""" + guardrail_id: str | None """Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'""" diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 137383c3483..963b7dde07b 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1682,6 +1682,36 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert rebuilds == [2] + @pytest.mark.asyncio + async def test_per_choice_inspection_labels_its_log_entries_with_the_choice_index(self) -> None: + """Without a label the per-choice entries of an n > 1 response cannot be told apart.""" + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class IndexRecorder(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(request_data.get("guardrail_choice_index")) + return inputs + + IndexRecorder.seen = [] + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("first", "second"))])] + request_data: Final = {} + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=IndexRecorder(guardrail_name="indexed"), + request_data=request_data, deliver_ended_stream_rewrites=True, + ) + assert IndexRecorder.seen == [0, 1] + assert "guardrail_choice_index" not in request_data + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From 5a2e78e55052c1ce8ef40d5bfebe86c65d46e4f5 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 18:33:51 +0100 Subject: [PATCH 6/7] fix(types): include choice_index in the guardrail audit fields --- litellm/types/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c6eeb41a9d1..d27c4bb62a6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3511,6 +3511,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset( "end_time", "duration", "masked_entity_count", + "choice_index", "guardrail_id", "policy_template", "detection_method", From 55545c693fe5f6183d74b1ca21c3eb00ce57a868 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 23:25:18 +0100 Subject: [PATCH 7/7] fix(guardrails): treat tool calls with no arguments yet as incomplete --- .../chat/guardrail_translation/handler.py | 81 ++++++-------- .../test_openai_guardrail_handler.py | 105 ++++++------------ 2 files changed, 71 insertions(+), 115 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index be698093fec..18c8b5aca9e 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -669,7 +669,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if pre_guardrail_tool_calls and len(model_response.choices) > 1 else (model_response,) ) - # Local import avoids a module-level cyclic import with the policy engine. from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite inspection_request_data: Final = request_data if request_data is not None else {} @@ -677,15 +676,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): prior_response: Final = inspection_request_data.get("response", missing) prior_responses: Final = inspection_request_data.get("responses", missing) prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) - # An undeliverable rewrite on one choice must not end the loop: the caller releases the - # original chunks when it catches this, so leaving later choices uninspected would let a - # request with n > 1 pair a redactable choice with a prohibited one and have the whole - # response delivered unchecked. Inspect every choice, then raise the first failure. undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response - # Labels this round's guardrail log entry and span with the choice it inspected. inspection_request_data["guardrail_choice_index"] = ( inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None ) @@ -709,9 +703,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if undeliverable is None: undeliverable = exc finally: - # Restore the caller's keys rather than overwriting them: the inspection loop owns - # these entries only for its own duration, and "responses" is not a key the - # non-streaming path sets at all. if prior_response is missing: inspection_request_data.pop("response", None) else: @@ -740,30 +731,49 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + @staticmethod + def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]: + for chunk in responses_so_far: + for choice in chunk.choices: + yield choice.index + + @staticmethod + def _streamed_argument_fragments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> Iterator[tuple[tuple[int, int], str]]: + for chunk in responses_so_far: + for choice in chunk.choices: + for call in choice.delta.tool_calls or (): + arguments = getattr(getattr(call, "function", None), "arguments", None) + yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else "" + + @staticmethod + def _parses_as_json(arguments: str) -> bool: + if not arguments: + return False + try: + json.loads(arguments) + except ValueError: + return False + return True + @staticmethod def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: """Whether every streamed tool call now carries arguments that parse as JSON. - Streamed argument fragments are only valid JSON once the last one has arrived, so this is - the completeness test: it holds back `{"ci` and lets `{"city": "Paris"}` through whether - the stream ended normally or was cut short by a disconnect. It reads the raw chunks so the - answer is available before deciding whether to rebuild. + A call that has announced itself but emitted no argument text yet is still in flight, so a + parallel call finishing first does not make the whole set complete. """ - fragments: Final[dict[tuple[int, int], list[str]]] = {} # mutable-ok: local accumulator - for chunk in responses_so_far: - for choice in chunk.choices: - for call in getattr(choice.delta, "tool_calls", None) or (): - arguments = getattr(getattr(call, "function", None), "arguments", None) - if isinstance(arguments, str) and arguments: - fragments.setdefault((choice.index, getattr(call, "index", 0)), []).append(arguments) + fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) if not fragments: return False - for parts in fragments.values(): - try: - json.loads("".join(parts)) - except ValueError: - return False - return True + joined: Final = MappingProxyType( + { + call: "".join(text for other, text in fragments if other == call) + for call in frozenset(call for call, _ in fragments) + } + ) + return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) @staticmethod def _rebuild_ended_stream_per_choice( @@ -884,15 +894,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - # Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is - # quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild. - # main rebuilds only on the ended path, so rebuild here only when this round consumes the - # assembled response -- to isolate choices, or to hand over complete tool calls once the - # stream has finished. A single-choice round in flight needs neither. stream_ended: Final = self._first_choice_has_finished(responses_so_far) - streamed_choice_indices: Final = frozenset( - choice.index for chunk in responses_so_far for choice in chunk.choices - ) + streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) assembled: Final = ( None @@ -910,14 +913,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for choice in assembled.choices ) - # Local import avoids a module-level cyclic import with the policy engine. from litellm.proxy.policy_engine.pipeline_executor import ( UndeliverableStreamRewrite, ) - # Same rule as the ended-stream inspection loop: an undeliverable rewrite on one - # choice must not leave the rest uninspected, because the caller releases the - # original chunks when it catches this. transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: @@ -960,12 +959,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) sink.record(merged_texts, merged_holdback) return - # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which - # main never handed to a guardrail: one that parses them either errors or judges - # arguments that do not exist yet. Completeness is what matters, not the finish reason: - # a stream cut short by a client disconnect carries whole arguments and no finish - # reason, and that call still has to be inspected. Send the calls once every one of - # them parses; the text of the earlier rounds is still inspected above. if assembled is not None and tool_arguments_complete: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 963b7dde07b..0a0bccceb71 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1434,15 +1434,11 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" if transform: - # The transform path seeds "response" and "responses" up front as the context the - # guardrail reads, so they are its own values rather than leftovers. assert request_data["responses"] is chunks restored: Final = request_data["response"] assert isinstance(restored, ModelResponse) assert tuple(choice.index for choice in restored.choices) == (0, 1) else: - # The ended-stream inspection loop owns these keys only while it runs; neither was in - # request_data beforehand, and the non-streaming path never sets "responses" at all. assert "response" not in request_data assert "responses" not in request_data @@ -1535,8 +1531,6 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], input_type: Literal["request", "response"], logging_obj: object = None, ) -> GenericGuardrailAPIInputs: - # Only the ended-stream inspection loop hands the guardrail an assembled - # "response"; the in-flight branch passes raw chunks under "responses" alone. type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse) return inputs @@ -1606,8 +1600,6 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: return inputs def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream: - # The text keeps the guardrail in play for the partial round; without any text the - # transform path has nothing to inspect and never calls it at all. return ModelResponseStream(choices=[StreamingChoices( index=0, delta=Delta(content=text, tool_calls=[{ "index": 0, "id": "call_0", "type": "function", @@ -1636,81 +1628,52 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] @pytest.mark.asyncio - async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None: - """Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length. - - main rebuilds only on the ended path; an in-flight single-choice round consumes nothing - from the assembled response, so it must not pay for one. - """ - from unittest.mock import patch - + async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None: from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - def chunk(text: str, finish_reason: str | None) -> ModelResponseStream: - return ModelResponseStream(choices=[StreamingChoices( - index=0, delta=Delta(content=text, tool_calls=[{ - "index": 0, "id": "call_0", "type": "function", - "function": {"name": "weather", "arguments": '{"c'}, - }]), finish_reason=finish_reason, - )]) - - handler: Final = OpenAIChatCompletionsHandler() - rebuilds: Final[list[int]] = [] - original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice - - def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy - rebuilds.append(len(responses_so_far)) - return original(responses_so_far, litellm_logging_obj) - - # The real attribute is a staticmethod, so the spy has to be one too: a plain function - # would be bound and receive the handler as its first argument. - with patch.object( - OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild) - ): - await handler.process_output_streaming_response( - responses_so_far=[chunk("looking", None)], - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), - ) - assert rebuilds == [] - - await handler.process_output_streaming_response( - responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")], - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), - ) - assert rebuilds == [2] - - @pytest.mark.asyncio - async def test_per_choice_inspection_labels_its_log_entries_with_the_choice_index(self) -> None: - """Without a label the per-choice entries of an n > 1 response cannot be told apart.""" - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - - class IndexRecorder(CustomGuardrail): + class RecordingGuardrail(CustomGuardrail): seen: list[object] = [] async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], input_type: Literal["request", "response"], logging_obj: object = None, ) -> GenericGuardrailAPIInputs: - type(self).seen.append(request_data.get("guardrail_choice_index")) + type(self).seen.append(inputs.get("tool_calls")) return inputs - IndexRecorder.seen = [] - chunks: Final = [ModelResponseStream(choices=[StreamingChoices( - index=index, delta=Delta(content=text, tool_calls=[{ - "index": 0, "id": f"call_{index}", "type": "function", - "function": {"name": "contact", "arguments": "{}"}, - }]), finish_reason="tool_calls", - ) for index, text in enumerate(("first", "second"))])] - request_data: Final = {} - await OpenAIChatCompletionsHandler().process_output_streaming_response( - responses_so_far=chunks, guardrail_to_apply=IndexRecorder(guardrail_name="indexed"), - request_data=request_data, deliver_ended_stream_rewrites=True, + def call(index: int, arguments: str) -> dict[str, object]: + return { + "index": index, "id": f"call_{index}", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + } + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]), + finish_reason=None, + )])] + await handler.process_output_streaming_response( + responses_so_far=one_still_empty, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), ) - assert IndexRecorder.seen == [0, 1] - assert "guardrail_choice_index" not in request_data + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + both_complete: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, + delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]), + finish_reason="tool_calls", + )])] + await handler.process_output_streaming_response( + responses_so_far=both_complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: