perf(guardrails): rebuild the stream only when a round consumes it

This commit is contained in:
Alessandro Pignati 2026-10-28 12:32:06 +01:00
parent fd3ad67c47
commit ff5644b3d8
2 changed files with 65 additions and 4 deletions

View file

@ -850,9 +850,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if self._streamed_tool_call_fingerprints(responses_so_far):
assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
resolved_request_data["response"] = assembled
if len(assembled.choices) > 1:
# Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is
# quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild.
# main rebuilds only on the ended path, so rebuild here only when this round consumes the
# assembled response -- to isolate choices, or to hand over complete tool calls once the
# stream has finished. A single-choice round in flight needs neither.
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
streamed_choice_indices: Final = frozenset(
choice.index for chunk in responses_so_far for choice in chunk.choices
)
assembled: Final = (
None
if len(streamed_choice_indices) == 1 and not stream_ended
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
)
if assembled is not None:
resolved_request_data["response"] = assembled
if assembled is not None and len(assembled.choices) > 1:
choice_rounds: Final = tuple(
(
choice,
@ -913,7 +927,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# main never handed to a guardrail: one that parses them either errors or judges
# arguments that do not exist yet. Send tool calls only once the stream has finished and
# the arguments are complete; the text of those rounds is still inspected above.
if self._first_choice_has_finished(responses_so_far):
if assembled is not None and stream_ended:
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
tuple(

View file

@ -1635,6 +1635,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
assert sent is not None
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
@pytest.mark.asyncio
async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None:
"""Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length.
main rebuilds only on the ended path; an in-flight single-choice round consumes nothing
from the assembled response, so it must not pay for one.
"""
from unittest.mock import patch
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def chunk(text: str, finish_reason: str | None) -> ModelResponseStream:
return ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": "call_0", "type": "function",
"function": {"name": "weather", "arguments": '{"c'},
}]), finish_reason=finish_reason,
)])
handler: Final = OpenAIChatCompletionsHandler()
rebuilds: Final[list[int]] = []
original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice
def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy
rebuilds.append(len(responses_so_far))
return original(responses_so_far, litellm_logging_obj)
# The real attribute is a staticmethod, so the spy has to be one too: a plain function
# would be bound and receive the handler as its first argument.
with patch.object(
OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild)
):
await handler.process_output_streaming_response(
responses_so_far=[chunk("looking", None)],
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert rebuilds == []
await handler.process_output_streaming_response(
responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")],
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert rebuilds == [2]
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None:
handler: Final = OpenAIChatCompletionsHandler()