mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
perf(guardrails): rebuild the stream only when a round consumes it
This commit is contained in:
parent
fd3ad67c47
commit
ff5644b3d8
2 changed files with 65 additions and 4 deletions
|
|
@ -850,9 +850,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if self._streamed_tool_call_fingerprints(responses_so_far):
|
||||
assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
resolved_request_data["response"] = assembled
|
||||
if len(assembled.choices) > 1:
|
||||
# Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is
|
||||
# quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild.
|
||||
# main rebuilds only on the ended path, so rebuild here only when this round consumes the
|
||||
# assembled response -- to isolate choices, or to hand over complete tool calls once the
|
||||
# stream has finished. A single-choice round in flight needs neither.
|
||||
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
streamed_choice_indices: Final = frozenset(
|
||||
choice.index for chunk in responses_so_far for choice in chunk.choices
|
||||
)
|
||||
assembled: Final = (
|
||||
None
|
||||
if len(streamed_choice_indices) == 1 and not stream_ended
|
||||
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
)
|
||||
if assembled is not None:
|
||||
resolved_request_data["response"] = assembled
|
||||
if assembled is not None and len(assembled.choices) > 1:
|
||||
choice_rounds: Final = tuple(
|
||||
(
|
||||
choice,
|
||||
|
|
@ -913,7 +927,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# main never handed to a guardrail: one that parses them either errors or judges
|
||||
# arguments that do not exist yet. Send tool calls only once the stream has finished and
|
||||
# the arguments are complete; the text of those rounds is still inspected above.
|
||||
if self._first_choice_has_finished(responses_so_far):
|
||||
if assembled is not None and stream_ended:
|
||||
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
|
|
|
|||
|
|
@ -1635,6 +1635,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
assert sent is not None
|
||||
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None:
|
||||
"""Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length.
|
||||
|
||||
main rebuilds only on the ended path; an in-flight single-choice round consumes nothing
|
||||
from the assembled response, so it must not pay for one.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
def chunk(text: str, finish_reason: str | None) -> ModelResponseStream:
|
||||
return ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": '{"c'},
|
||||
}]), finish_reason=finish_reason,
|
||||
)])
|
||||
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
rebuilds: Final[list[int]] = []
|
||||
original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice
|
||||
|
||||
def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy
|
||||
rebuilds.append(len(responses_so_far))
|
||||
return original(responses_so_far, litellm_logging_obj)
|
||||
|
||||
# The real attribute is a staticmethod, so the spy has to be one too: a plain function
|
||||
# would be bound and receive the handler as its first argument.
|
||||
with patch.object(
|
||||
OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild)
|
||||
):
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=[chunk("looking", None)],
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert rebuilds == []
|
||||
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")],
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert rebuilds == [2]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None:
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue