mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(guardrails): write tool-call rewrites back on multi-choice chat streams
The ended-stream write-back grouped tool-call fragments by tool index alone, so it refused every stream with more than one choice. The pipeline then released the original chunks, undoing text redactions on n>1 streams as soon as a guardrail also rewrote a tool call. Fragments are now keyed by (choice index, tool index), the key stream_chunk_builder already uses, and the refusal is gone
This commit is contained in:
parent
8ea78482cc
commit
f394b968c5
2 changed files with 57 additions and 46 deletions
|
|
@ -1225,20 +1225,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
def _function_tool_call_fragments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]:
|
||||
"""Group the stream's function tool-call fragments by their tool-call index, in
|
||||
the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping
|
||||
only the indices the builder keeps (an id and a name somewhere in the stream)."""
|
||||
fragments: Final = tuple(
|
||||
tool_call
|
||||
for response in responses_so_far
|
||||
for choice in response.choices
|
||||
for tool_call in choice.delta.tool_calls or ()
|
||||
if isinstance(tool_call, ChatCompletionDeltaToolCall)
|
||||
)
|
||||
identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id)
|
||||
named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name)
|
||||
"""Group the stream's function tool-call fragments by (choice index, tool-call index), in
|
||||
the order the per-choice rebuild lists the tool calls, keeping only the calls
|
||||
``stream_chunk_builder`` keeps (an id and a name somewhere in the stream)."""
|
||||
fragments: Final = tuple(_keyed_function_tool_call_fragments(responses_so_far))
|
||||
identified: Final = frozenset(key for key, fragment in fragments if fragment.id)
|
||||
named: Final = frozenset(key for key, fragment in fragments if fragment.function.name)
|
||||
return tuple(
|
||||
tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named)
|
||||
tuple(fragment for key, fragment in fragments if key == kept) for kept in sorted(identified & named)
|
||||
)
|
||||
|
||||
def _write_ended_stream_tool_call_rewrites(
|
||||
|
|
@ -1251,25 +1245,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
"""Write ended-stream guardrail tool-call rewrites back across the buffered
|
||||
chunks: the rewritten name and full arguments land in the tool call's first
|
||||
fragment and the arguments of its later fragments are blanked, mirroring the
|
||||
text write-back. A rewrite on a stream carrying more than one distinct choice
|
||||
index, or whose fragments do not line up with the rebuilt tool calls, is
|
||||
reported as undeliverable, so the pipeline executor discards it and releases
|
||||
the original chunks."""
|
||||
text write-back. A rewrite whose fragments do not line up with the rebuilt tool
|
||||
calls is reported as undeliverable, so the pipeline executor discards it and
|
||||
releases the original chunks."""
|
||||
post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response)
|
||||
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
|
||||
return
|
||||
stream_choice_indices: Final = frozenset(
|
||||
choice.index for response in responses_so_far for choice in response.choices
|
||||
)
|
||||
fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far)
|
||||
if len(stream_choice_indices) != 1:
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
raise UndeliverableStreamRewrite(
|
||||
guardrail_name,
|
||||
f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written "
|
||||
"back on single-choice streams",
|
||||
)
|
||||
if len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
|
|
@ -1555,6 +1537,16 @@ def _streamed_server_tool_calls(
|
|||
yield (choice.index, server_tool_call["id"]), server_tool_call
|
||||
|
||||
|
||||
def _keyed_function_tool_call_fragments(
|
||||
responses: Sequence["ModelResponseStream"],
|
||||
) -> Iterator[tuple[tuple[int, int], ChatCompletionDeltaToolCall]]:
|
||||
for response in responses:
|
||||
for choice in response.choices:
|
||||
for tool_call in choice.delta.tool_calls or ():
|
||||
if isinstance(tool_call, ChatCompletionDeltaToolCall):
|
||||
yield (choice.index, tool_call.index), tool_call
|
||||
|
||||
|
||||
def _choice_server_tool_calls(
|
||||
calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int
|
||||
) -> tuple[ChatCompletionAssistantToolCall, ...]:
|
||||
|
|
|
|||
|
|
@ -1559,7 +1559,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
assert chunks[4].usage.completion_tokens == 7
|
||||
|
||||
@staticmethod
|
||||
def _two_choice_tool_call_stream_chunks() -> list:
|
||||
def _two_choice_tool_call_stream_chunks(leading_texts: tuple[str, str] | None = None) -> list:
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionDeltaToolCall,
|
||||
Delta,
|
||||
|
|
@ -1569,7 +1569,10 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
|
||||
def chunk(
|
||||
choice_index: int, tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None
|
||||
choice_index: int,
|
||||
tool_call: ChatCompletionDeltaToolCall | None,
|
||||
finish_reason: Optional[str] = None,
|
||||
content: str | None = None,
|
||||
) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id="chatcmpl-123",
|
||||
|
|
@ -1579,7 +1582,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
choices=[
|
||||
StreamingChoices(
|
||||
index=choice_index,
|
||||
delta=Delta(tool_calls=[tool_call] if tool_call else None),
|
||||
delta=Delta(content=content, tool_calls=[tool_call] if tool_call else None),
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
],
|
||||
|
|
@ -1590,7 +1593,11 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments)
|
||||
)
|
||||
|
||||
text_chunks: Final = (
|
||||
[chunk(index, None, content=text) for index, text in enumerate(leading_texts)] if leading_texts else []
|
||||
)
|
||||
return [
|
||||
*text_chunks,
|
||||
chunk(0, fragment("", name="lookup_fruit", call_id="call_1")),
|
||||
chunk(1, fragment("", name="lookup_fruit", call_id="call_2")),
|
||||
chunk(0, fragment('{"fruit": "pers')),
|
||||
|
|
@ -1639,25 +1646,37 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
async def test_deliver_ended_stream_rewrites_each_choices_text_and_tool_call(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
chunks = self._two_choice_tool_call_stream_chunks()
|
||||
chunks = self._two_choice_tool_call_stream_chunks(leading_texts=("look up ", "check "))
|
||||
|
||||
with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised:
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
assert raised.value.guardrail_name == "test"
|
||||
assert raised.value.reason == (
|
||||
"the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams"
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
|
||||
assert [
|
||||
(
|
||||
c.choices[0].index,
|
||||
c.choices[0].delta.content,
|
||||
[t.function.arguments for t in c.choices[0].delta.tool_calls or ()],
|
||||
)
|
||||
for c in chunks
|
||||
] == [
|
||||
(0, "LOOK UP ", []),
|
||||
(1, "CHECK ", []),
|
||||
(0, None, ['{"fruit": "PERSIMMON"}']),
|
||||
(1, None, ['{"fruit": "DURIAN"}']),
|
||||
(0, None, [""]),
|
||||
(1, None, [""]),
|
||||
(0, None, [""]),
|
||||
(1, None, [""]),
|
||||
(0, None, []),
|
||||
(1, None, []),
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self):
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue