fix(guardrails): write tool-call rewrites back on multi-choice chat streams

The ended-stream write-back grouped tool-call fragments by tool index alone, so it
refused every stream with more than one choice. The pipeline then released the
original chunks, undoing text redactions on n>1 streams as soon as a guardrail also
rewrote a tool call. Fragments are now keyed by (choice index, tool index), the key
stream_chunk_builder already uses, and the refusal is gone
This commit is contained in:
Jesus Velarde 2026-11-13 16:01:27 -08:00
parent 8ea78482cc
commit f394b968c5
2 changed files with 57 additions and 46 deletions

View file

@ -1225,20 +1225,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
def _function_tool_call_fragments(
responses_so_far: Sequence["ModelResponseStream"],
) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]:
"""Group the stream's function tool-call fragments by their tool-call index, in
the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping
only the indices the builder keeps (an id and a name somewhere in the stream)."""
fragments: Final = tuple(
tool_call
for response in responses_so_far
for choice in response.choices
for tool_call in choice.delta.tool_calls or ()
if isinstance(tool_call, ChatCompletionDeltaToolCall)
)
identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id)
named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name)
"""Group the stream's function tool-call fragments by (choice index, tool-call index), in
the order the per-choice rebuild lists the tool calls, keeping only the calls
``stream_chunk_builder`` keeps (an id and a name somewhere in the stream)."""
fragments: Final = tuple(_keyed_function_tool_call_fragments(responses_so_far))
identified: Final = frozenset(key for key, fragment in fragments if fragment.id)
named: Final = frozenset(key for key, fragment in fragments if fragment.function.name)
return tuple(
tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named)
tuple(fragment for key, fragment in fragments if key == kept) for kept in sorted(identified & named)
)
def _write_ended_stream_tool_call_rewrites(
@ -1251,25 +1245,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
"""Write ended-stream guardrail tool-call rewrites back across the buffered
chunks: the rewritten name and full arguments land in the tool call's first
fragment and the arguments of its later fragments are blanked, mirroring the
text write-back. A rewrite on a stream carrying more than one distinct choice
index, or whose fragments do not line up with the rebuilt tool calls, is
reported as undeliverable, so the pipeline executor discards it and releases
the original chunks."""
text write-back. A rewrite whose fragments do not line up with the rebuilt tool
calls is reported as undeliverable, so the pipeline executor discards it and
releases the original chunks."""
post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response)
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
stream_choice_indices: Final = frozenset(
choice.index for response in responses_so_far for choice in response.choices
)
fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far)
if len(stream_choice_indices) != 1:
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written "
"back on single-choice streams",
)
if len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
@ -1555,6 +1537,16 @@ def _streamed_server_tool_calls(
yield (choice.index, server_tool_call["id"]), server_tool_call
def _keyed_function_tool_call_fragments(
responses: Sequence["ModelResponseStream"],
) -> Iterator[tuple[tuple[int, int], ChatCompletionDeltaToolCall]]:
for response in responses:
for choice in response.choices:
for tool_call in choice.delta.tool_calls or ():
if isinstance(tool_call, ChatCompletionDeltaToolCall):
yield (choice.index, tool_call.index), tool_call
def _choice_server_tool_calls(
calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int
) -> tuple[ChatCompletionAssistantToolCall, ...]:

View file

@ -1559,7 +1559,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
assert chunks[4].usage.completion_tokens == 7
@staticmethod
def _two_choice_tool_call_stream_chunks() -> list:
def _two_choice_tool_call_stream_chunks(leading_texts: tuple[str, str] | None = None) -> list:
from litellm.types.utils import (
ChatCompletionDeltaToolCall,
Delta,
@ -1569,7 +1569,10 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
)
def chunk(
choice_index: int, tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None
choice_index: int,
tool_call: ChatCompletionDeltaToolCall | None,
finish_reason: Optional[str] = None,
content: str | None = None,
) -> ModelResponseStream:
return ModelResponseStream(
id="chatcmpl-123",
@ -1579,7 +1582,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
choices=[
StreamingChoices(
index=choice_index,
delta=Delta(tool_calls=[tool_call] if tool_call else None),
delta=Delta(content=content, tool_calls=[tool_call] if tool_call else None),
finish_reason=finish_reason,
)
],
@ -1590,7 +1593,11 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments)
)
text_chunks: Final = (
[chunk(index, None, content=text) for index, text in enumerate(leading_texts)] if leading_texts else []
)
return [
*text_chunks,
chunk(0, fragment("", name="lookup_fruit", call_id="call_1")),
chunk(1, fragment("", name="lookup_fruit", call_id="call_2")),
chunk(0, fragment('{"fruit": "pers')),
@ -1639,25 +1646,37 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')]
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
async def test_deliver_ended_stream_rewrites_each_choices_text_and_tool_call(self):
handler = OpenAIChatCompletionsHandler()
chunks = self._two_choice_tool_call_stream_chunks()
chunks = self._two_choice_tool_call_stream_chunks(leading_texts=("look up ", "check "))
with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised:
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert raised.value.guardrail_name == "test"
assert raised.value.reason == (
"the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams"
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert [
(
c.choices[0].index,
c.choices[0].delta.content,
[t.function.arguments for t in c.choices[0].delta.tool_calls or ()],
)
for c in chunks
] == [
(0, "LOOK UP ", []),
(1, "CHECK ", []),
(0, None, ['{"fruit": "PERSIMMON"}']),
(1, None, ['{"fruit": "DURIAN"}']),
(0, None, [""]),
(1, None, [""]),
(0, None, [""]),
(1, None, [""]),
(0, None, []),
(1, None, []),
]
@pytest.mark.asyncio
async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self):
handler = OpenAIChatCompletionsHandler()