From ff5644b3d873bfce95e6ff300432631d135a2005 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:32:06 +0100 Subject: [PATCH] perf(guardrails): rebuild the stream only when a round consumes it --- .../chat/guardrail_translation/handler.py | 22 +++++++-- .../test_openai_guardrail_handler.py | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index ef9c1eb34ab..a43c6b193b6 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -850,9 +850,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - resolved_request_data["response"] = assembled - if len(assembled.choices) > 1: + # Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is + # quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild. + # main rebuilds only on the ended path, so rebuild here only when this round consumes the + # assembled response -- to isolate choices, or to hand over complete tool calls once the + # stream has finished. A single-choice round in flight needs neither. + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + streamed_choice_indices: Final = frozenset( + choice.index for chunk in responses_so_far for choice in chunk.choices + ) + assembled: Final = ( + None + if len(streamed_choice_indices) == 1 and not stream_ended + else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + ) + if assembled is not None: + resolved_request_data["response"] = assembled + if assembled is not None and len(assembled.choices) > 1: choice_rounds: Final = tuple( ( choice, @@ -913,7 +927,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # main never handed to a guardrail: one that parses them either errors or judges # arguments that do not exist yet. Send tool calls only once the stream has finished and # the arguments are complete; the text of those rounds is still inspected above. - if self._first_choice_has_finished(responses_so_far): + if assembled is not None and stream_ended: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( tuple( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index a503bbf386e..137383c3483 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1635,6 +1635,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + @pytest.mark.asyncio + async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None: + """Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length. + + main rebuilds only on the ended path; an in-flight single-choice round consumes nothing + from the assembled response, so it must not pay for one. + """ + from unittest.mock import patch + + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(text: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": '{"c'}, + }]), finish_reason=finish_reason, + )]) + + handler: Final = OpenAIChatCompletionsHandler() + rebuilds: Final[list[int]] = [] + original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice + + def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy + rebuilds.append(len(responses_so_far)) + return original(responses_so_far, litellm_logging_obj) + + # The real attribute is a staticmethod, so the spy has to be one too: a plain function + # would be bound and receive the handler as its first argument. + with patch.object( + OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild) + ): + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None)], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [] + + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [2] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler()