From c4f7c3f66da347201c1bcbd9c1bc982dfb96db77 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sat, 31 Oct 2026 23:50:57 +0100 Subject: [PATCH 1/8] perf(guardrails): group streamed tool arguments in one pass Joining the fragments rescanned all of them once per distinct tool call, so a sampled round cost the number of calls times the length of the stream and at n=2 grew with the square of the streamed arguments. Group them in a single pass, and narrow the pruned response rather than the raw one in the per-choice rounds. --- .../chat/guardrail_translation/handler.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 9f9485692c2..3f0500dbf0a 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -19,7 +19,7 @@ import json import time import uuid from collections.abc import Iterator, Mapping, Sequence -from itertools import chain +from itertools import chain, groupby from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast @@ -762,13 +762,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _joined_streamed_arguments( responses_so_far: Sequence["ModelResponseStream"], ) -> Mapping[tuple[int, int], str]: - """The argument text streamed so far per (choice index, tool call index).""" - fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) + """The argument text streamed so far per (choice index, tool call index). + + Grouped in one pass. Rescanning every fragment once per distinct call made a sampled + round cost the number of calls times the length of the stream, which at n=2 grows with + the square of the streamed arguments. + """ + fragments: Final = sorted( + OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far), + key=lambda fragment: fragment[0], + ) return MappingProxyType( - { - call: "".join(text for other, text in fragments if other == call) - for call in frozenset(call for call, _ in fragments) - } + {call: "".join(text for _, text in group) for call, group in groupby(fragments, key=lambda f: f[0])} ) @staticmethod @@ -932,20 +937,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) - if assembled is not None: - resolved_request_data["response"] = ( - assembled - if tool_arguments_complete - else self._without_unfinished_tool_calls(assembled, responses_so_far) - ) - if assembled is not None and len(assembled.choices) > 1: + inspected: Final = ( + None + if assembled is None + else assembled + if tool_arguments_complete + else self._without_unfinished_tool_calls(assembled, responses_so_far) + ) + if inspected is not None: + resolved_request_data["response"] = inspected + if inspected is not None and len(inspected.choices) > 1: choice_rounds: Final = tuple( ( choice, StreamTransformSink(), [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], ) - for choice in assembled.choices + for choice in inspected.choices ) from litellm.proxy.policy_engine.pipeline_executor import ( UndeliverableStreamRewrite, @@ -954,7 +962,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: - resolved_request_data["response"] = assembled.model_copy( + resolved_request_data["response"] = inspected.model_copy( update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks From 215e0143110fa458cd37ed7b5065d32b23e109bf Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sat, 31 Oct 2026 23:59:41 +0100 Subject: [PATCH 2/8] refactor(guardrails): match unfinished tool calls by their announced id Pruning matched calls by their position in the rebuilt list, which need not follow the order the stream gave them, and did it by deep-copying the response and writing into the model. Match on the id each call announced, build the pruned copies with model_copy, and return the response untouched when every call finished. --- .../chat/guardrail_translation/handler.py | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3f0500dbf0a..1ac7f6ebb0a 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -14,7 +14,6 @@ Pattern Overview: This pattern can be replicated for other message formats (e.g., Anthropic). """ -import copy import json import time import uuid @@ -788,27 +787,44 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return False return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) + @staticmethod + def _streamed_call_ids(responses_so_far: Sequence["ModelResponseStream"]) -> Mapping[tuple[int, int], str]: + """The id each (choice index, tool call index) announced itself with.""" + choices: Final = chain.from_iterable(chunk.choices for chunk in responses_so_far) + calls: Final = chain.from_iterable( + ((choice.index, call) for call in choice.delta.tool_calls or ()) for choice in choices + ) + return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id}) + + @staticmethod + def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices": + kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished] + message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None})) + return choice.model_copy(update=MappingProxyType({"message": message})) + @staticmethod def _without_unfinished_tool_calls( assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"] ) -> "ModelResponse": """Copy of the rebuilt response without the calls still in flight when the stream ended. - Rebuilding gives a call that never streamed any argument text an empty JSON object, which - reads to a guardrail as a real call taking no arguments rather than an unfinished one. + Rebuilding gives a call that never streamed complete arguments an empty JSON object, which + reads to a guardrail as a real call taking no arguments rather than an unfinished one. The + calls are matched by the id they announced, since the rebuilt order need not follow the + order the stream gave them. """ joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) - pruned: Final = copy.deepcopy(assembled) - for choice in pruned.choices: - streamed = choice.message.tool_calls or () - kept = [ - call - for position, call in enumerate(streamed) - if OpenAIChatCompletionsHandler._parses_as_json(joined.get((choice.index, position), "")) - ] - if len(kept) != len(streamed): - choice.message.tool_calls = kept or None - return pruned + unfinished: Final = frozenset( + call_id + for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items() + if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, "")) + ) + if not unfinished: + return assembled + pruned: Final = [ + OpenAIChatCompletionsHandler._choice_without(choice, unfinished) for choice in assembled.choices + ] + return assembled.model_copy(update=MappingProxyType({"choices": pruned})) @staticmethod def _rebuild_ended_stream_per_choice( From 175ddd4faf21ce61e03deedfb896402bd2aa26d4 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:08:39 +0100 Subject: [PATCH 3/8] fix(guardrails): leave request_data as the round found it A diff-mode round wrote response, responses and the choice index into the caller's request_data and put back its own values instead of what was there, so a stale response outlived the round and later hooks read it. Record what each path found and restore it, including when the round raises. --- .../chat/guardrail_translation/handler.py | 250 ++++++++++-------- .../test_openai_guardrail_handler.py | 37 ++- 2 files changed, 167 insertions(+), 120 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 1ac7f6ebb0a..75d0500324f 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -609,8 +609,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # create a standalone dict (SDK / direct-call path). if request_data is None: request_data = {"responses": responses_so_far} + owes_responses = False # rebind-ok: the standalone dict is ours to keep else: - if "responses" not in request_data: + owes_responses = "responses" not in request_data # rebind-ok: tracks the key this scan adds + if owes_responses: request_data["responses"] = responses_so_far self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) @@ -621,12 +623,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # Include model information from the first response if available if responses_so_far and hasattr(responses_so_far[0], "model") and responses_so_far[0].model: inputs["model"] = responses_so_far[0].model - guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( - inputs=inputs, - request_data=request_data, - input_type="response", - logging_obj=litellm_logging_obj, - ) + try: + guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( + inputs=inputs, + request_data=request_data, + input_type="response", + logging_obj=litellm_logging_obj, + ) + finally: + if owes_responses: + request_data.pop("responses", None) guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) @@ -939,125 +945,145 @@ class OpenAIChatCompletionsHandler(BaseTranslation): resolved_request_data: Final[dict[str, object]] = ( request_data if request_data is not None else {"responses": responses_so_far} ) + missing: Final = object() + prior_response: Final = resolved_request_data.get("response", missing) + prior_responses: Final = resolved_request_data.get("responses", missing) + prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing) if "responses" not in resolved_request_data: resolved_request_data["responses"] = responses_so_far self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) - - inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) - if self._streamed_tool_call_fingerprints(responses_so_far): - stream_ended: Final = self._first_choice_has_finished(responses_so_far) - streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) - tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) - assembled: Final = ( - None - if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete - else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - ) - inspected: Final = ( - None - if assembled is None - else assembled - if tool_arguments_complete - else self._without_unfinished_tool_calls(assembled, responses_so_far) - ) - if inspected is not None: - resolved_request_data["response"] = inspected - if inspected is not None and len(inspected.choices) > 1: - choice_rounds: Final = tuple( - ( - choice, - StreamTransformSink(), - [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], - ) - for choice in inspected.choices + try: + inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) + if self._streamed_tool_call_fingerprints(responses_so_far): + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) + tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) + assembled: Final = ( + None + if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete + else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) - from litellm.proxy.policy_engine.pipeline_executor import ( - UndeliverableStreamRewrite, + inspected: Final = ( + None + if assembled is None + else assembled + if tool_arguments_complete + else self._without_unfinished_tool_calls(assembled, responses_so_far) ) - - transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure - try: - for choice, choice_sink, choice_chunks in choice_rounds: - resolved_request_data["response"] = inspected.model_copy( - update=MappingProxyType({"choices": [choice]}) + if inspected is not None: + resolved_request_data["response"] = inspected + if inspected is not None and len(inspected.choices) > 1: + choice_rounds: Final = tuple( + ( + choice, + StreamTransformSink(), + [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], ) - resolved_request_data["responses"] = choice_chunks - resolved_request_data["guardrail_choice_index"] = choice.index - try: - await self._process_streaming_transform( - responses_so_far=choice_chunks, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=resolved_request_data, - sink=choice_sink, + for choice in inspected.choices + ) + from litellm.proxy.policy_engine.pipeline_executor import ( + UndeliverableStreamRewrite, + ) + + transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: first failure + try: + for choice, choice_sink, choice_chunks in choice_rounds: + resolved_request_data["response"] = inspected.model_copy( + update=MappingProxyType({"choices": [choice]}) + ) + resolved_request_data["responses"] = choice_chunks + resolved_request_data["guardrail_choice_index"] = choice.index + try: + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + except UndeliverableStreamRewrite as exc: + if transform_undeliverable is None: + transform_undeliverable = exc + finally: + resolved_request_data["response"] = assembled + resolved_request_data["responses"] = responses_so_far + resolved_request_data.pop("guardrail_choice_index", None) + if transform_undeliverable is not None: + raise transform_undeliverable + merged_texts: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds ) - except UndeliverableStreamRewrite as exc: - if transform_undeliverable is None: - transform_undeliverable = exc - finally: - resolved_request_data["response"] = assembled - resolved_request_data["responses"] = responses_so_far - resolved_request_data.pop("guardrail_choice_index", None) - if transform_undeliverable is not None: - raise transform_undeliverable - merged_texts: Final = MappingProxyType( - dict( - chain.from_iterable( - choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds ) ) - ) - merged_holdback: Final = MappingProxyType( - dict( - chain.from_iterable( - choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds + merged_holdback: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds + ) ) ) - ) - sink.record(merged_texts, merged_holdback) - return - if assembled is not None and tool_arguments_complete: - tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) - inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( - tuple( - MappingProxyType({"index": index, **converted}) - for index, tool_call in enumerate(tool_calls) - if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + sink.record(merged_texts, merged_holdback) + return + if assembled is not None and tool_arguments_complete: + tool_calls: Final = chain.from_iterable( + choice.message.tool_calls or () for choice in assembled.choices ) - ) - if responses_so_far and getattr(responses_so_far[0], "model", None): - inputs["model"] = responses_so_far[0].model - guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( - inputs=inputs, - request_data=resolved_request_data, - input_type="response", - logging_obj=litellm_logging_obj, - ) - - returned_texts = guardrailed_inputs.get("texts") - # No "texts" key means the guardrail made no change (action NONE): the raw - # accumulated text is the guardrailed text. A present-but-shorter list is a - # guardrail contract violation; those choices are omitted below (withheld, - # not emitted raw) so a malformed response fails closed instead of leaking. - if returned_texts is None: - returned_texts = texts_to_check - elif len(returned_texts) < len(texts_to_check): - verbose_proxy_logger.warning( - "OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the " - "streaming transform path; withholding the unmatched choices to fail closed.", - len(returned_texts), - len(texts_to_check), + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) + ) + if responses_so_far and getattr(responses_so_far[0], "model", None): + inputs["model"] = responses_so_far[0].model + guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( + inputs=inputs, + request_data=resolved_request_data, + input_type="response", + logging_obj=litellm_logging_obj, ) - holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () - transformed_texts: Final = MappingProxyType( - {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} - ) - transformed_holdback: Final = MappingProxyType( - {indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)} - ) - sink.record(transformed_texts, transformed_holdback) + returned_texts = guardrailed_inputs.get("texts") + # No "texts" key means the guardrail made no change (action NONE): the raw + # accumulated text is the guardrailed text. A present-but-shorter list is a + # guardrail contract violation; those choices are omitted below (withheld, + # not emitted raw) so a malformed response fails closed instead of leaking. + if returned_texts is None: + returned_texts = texts_to_check + elif len(returned_texts) < len(texts_to_check): + verbose_proxy_logger.warning( + "OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the " + "streaming transform path; withholding the unmatched choices to fail closed.", + len(returned_texts), + len(texts_to_check), + ) + + holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () + transformed_texts: Final = MappingProxyType( + {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} + ) + transformed_holdback: Final = MappingProxyType( + { + indices[i]: coerce_stream_holdback_value(holdback[i]) + for i in range(len(indices)) + if i < len(holdback) + } + ) + sink.record(transformed_texts, transformed_holdback) + finally: + for key, prior in ( + ("response", prior_response), + ("responses", prior_responses), + ("guardrail_choice_index", prior_choice_index), + ): + if prior is missing: + resolved_request_data.pop(key, None) + else: + resolved_request_data[key] = prior def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 3de123a4241..b868e70a902 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1433,14 +1433,8 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" - if transform: - assert request_data["responses"] is chunks - restored: Final = request_data["response"] - assert isinstance(restored, ModelResponse) - assert tuple(choice.index for choice in restored.choices) == (0, 1) - else: - assert "response" not in request_data - assert "responses" not in request_data + assert "response" not in request_data + assert "responses" not in request_data @pytest.mark.asyncio async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None: @@ -1675,6 +1669,33 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_a_transform_round_leaves_the_callers_request_data_as_it_found_it(self) -> None: + """The round owns these keys only while it runs; what it leaves behind is read by later hooks.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + handler: Final = OpenAIChatCompletionsHandler() + request_data: Final[dict[str, object]] = {"model": "gpt-4"} + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": '{"city": "Paris"}'}}, + ]), + finish_reason="tool_calls", + )])] + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + request_data=request_data, + stream_transform_sink=StreamTransformSink(), + ) + assert "response" not in request_data + assert "responses" not in request_data + assert "guardrail_choice_index" not in request_data + @pytest.mark.asyncio async def test_an_unfinished_parallel_call_is_kept_out_of_what_the_guardrail_reads(self) -> None: """A call that only announced its name must not reach the guardrail, by either route.""" From e53c6fc8bf35c4cd7fea0384efcba49aa8ca67bd Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:17:24 +0100 Subject: [PATCH 4/8] fix(guardrails): let only the round set the choice index label The label was read straight from request_data, so a caller could put guardrail_choice_index in the request body and have it logged as the choice a guardrail inspected. Drop any incoming value when a round starts, and restore the caller's own on the way out. --- .../chat/guardrail_translation/handler.py | 2 ++ .../test_openai_guardrail_handler.py | 36 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 75d0500324f..01cf518e04b 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -682,6 +682,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): prior_response: Final = inspection_request_data.get("response", missing) prior_responses: Final = inspection_request_data.get("responses", missing) prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) + inspection_request_data.pop("guardrail_choice_index", None) undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: @@ -949,6 +950,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): prior_response: Final = resolved_request_data.get("response", missing) prior_responses: Final = resolved_request_data.get("responses", missing) prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing) + resolved_request_data.pop("guardrail_choice_index", None) if "responses" not in resolved_request_data: resolved_request_data["responses"] = responses_so_far self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index b868e70a902..73abb328848 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1669,6 +1669,42 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_a_caller_cannot_set_the_choice_index_label(self) -> None: + """The label says which choice the round inspected, so only the round may set it.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class LabelReadingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(request_data.get("guardrail_choice_index")) + return inputs + + LabelReadingGuardrail.seen = [] + request_data: Final[dict[str, object]] = {"model": "gpt-4", "guardrail_choice_index": 7} + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": '{"city": "Paris"}'}}, + ]), + finish_reason="tool_calls", + )])] + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=LabelReadingGuardrail(guardrail_name="label"), + litellm_logging_obj=None, + request_data=request_data, + stream_transform_sink=StreamTransformSink(), + ) + assert LabelReadingGuardrail.seen == [None] + assert request_data["guardrail_choice_index"] == 7 + @pytest.mark.asyncio async def test_a_transform_round_leaves_the_callers_request_data_as_it_found_it(self) -> None: """The round owns these keys only while it runs; what it leaves behind is read by later hooks.""" From e54d4b8d12053d8dbfa72a9024e20dbdb2c8d644 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:24:57 +0100 Subject: [PATCH 5/8] fix(guardrails): label the diff-mode round with the choice it inspected The ended-stream path labelled every inspection, including a single choice, while a diff-mode round only labelled the per-choice rounds of an n>1 stream. A single-choice round logged no choice at all. Label it with the choice its texts came from. --- litellm/llms/openai/chat/guardrail_translation/handler.py | 2 ++ .../chat/guardrail_translation/test_openai_guardrail_handler.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 01cf518e04b..bd76a2ae9f2 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -951,6 +951,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): prior_responses: Final = resolved_request_data.get("responses", missing) prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing) resolved_request_data.pop("guardrail_choice_index", None) + if len(indices) == 1: + resolved_request_data["guardrail_choice_index"] = indices[0] if "responses" not in resolved_request_data: resolved_request_data["responses"] = responses_so_far self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 73abb328848..d8d06f34519 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1702,7 +1702,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: request_data=request_data, stream_transform_sink=StreamTransformSink(), ) - assert LabelReadingGuardrail.seen == [None] + assert LabelReadingGuardrail.seen == [0] assert request_data["guardrail_choice_index"] == 7 @pytest.mark.asyncio From f8a9c28108057077d09b5c692df131182eb587cd Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:32:33 +0100 Subject: [PATCH 6/8] test(guardrails): cover partial tool arguments at n>1 An argument fragment that is still half an object cannot be judged, and the id-based pruning already keeps it away from the guardrail. Lock that in for a stream with more than one choice, where the per-choice rounds narrow the inspected response. --- .../test_openai_guardrail_handler.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index d8d06f34519..57931dcad67 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1669,6 +1669,49 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None: + """Half a JSON object is not a call a guardrail can judge, at any number of choices.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(request_data.get("response")) + return inputs + + def choice(index: int, arguments: str, label: str) -> StreamingChoices: + return StreamingChoices( + index=index, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": "call_" + label, "type": "function", + "function": {"name": "weather", "arguments": arguments}}, + ]), + finish_reason="tool_calls", + ) + + RecordingGuardrail.seen = [] + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=[ModelResponseStream(choices=[ + choice(0, '{"city": "Pa', "partial"), choice(1, '{"city": "Rome"}', "rome"), + ])], + guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"), + litellm_logging_obj=None, request_data={}, + stream_transform_sink=StreamTransformSink(), + ) + arguments: Final = tuple( + tool_call.function.arguments + for response in RecordingGuardrail.seen + for single in getattr(response, "choices", None) or () + for tool_call in single.message.tool_calls or () + ) + assert arguments == ('{"city": "Rome"}',) + @pytest.mark.asyncio async def test_a_caller_cannot_set_the_choice_index_label(self) -> None: """The label says which choice the round inspected, so only the round may set it.""" From 0ea3b898c1213aa430b7169d6a70cbb13d52b0b4 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:43:43 +0100 Subject: [PATCH 7/8] perf(guardrails): buy per-choice context only when there is a call to judge At n>1 every sampled round rebuilt the stream and ran one guardrail call per choice, even while no tool call had complete arguments, so a vendor was billed n times a round for context that carried nothing to judge. Rebuild once the stream ends or the arguments parse; until then the choices share one round, with their texts still kept apart. --- .../chat/guardrail_translation/handler.py | 9 +---- .../test_openai_guardrail_handler.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index bd76a2ae9f2..3d45feb0eab 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -738,12 +738,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) - @staticmethod - def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]: - for chunk in responses_so_far: - for choice in chunk.choices: - yield choice.index - @staticmethod def _streamed_argument_fragments( responses_so_far: Sequence["ModelResponseStream"], @@ -960,11 +954,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): stream_ended: Final = self._first_choice_has_finished(responses_so_far) - streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) assembled: Final = ( None - if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete + if not stream_ended and not tool_arguments_complete else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) inspected: Final = ( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 57931dcad67..f139e2c90fc 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1669,6 +1669,42 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_a_round_with_nothing_to_judge_costs_one_call_at_n2(self) -> None: + """Per-choice rounds buy tool-call context. While no call is complete there is none to buy, + and charging a vendor once per choice for every sampled round buys nothing.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class CountingGuardrail(CustomGuardrail): + calls: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).calls.append(inputs.get("texts")) + return inputs + + def choice(index: int, arguments: str) -> StreamingChoices: + return StreamingChoices( + index=index, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "weather", "arguments": arguments}}, + ]), + finish_reason=None, + ) + + CountingGuardrail.calls = [] + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=[ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"ci')])], + guardrail_to_apply=CountingGuardrail(guardrail_name="counting"), + litellm_logging_obj=None, request_data={}, + stream_transform_sink=StreamTransformSink(), + ) + assert len(CountingGuardrail.calls) == 1 + @pytest.mark.asyncio async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None: """Half a JSON object is not a call a guardrail can judge, at any number of choices.""" From 1a4f6c5542572deb2cdd268e8f8e54f62eb614ab Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sun, 1 Nov 2026 22:53:39 +0100 Subject: [PATCH 8/8] fix(guardrails): rebuild a stream that carried no choices instead of failing Rebuilding one choice index at a time unpacked the first rebuilt choice, so a stream that ended carrying only usage left nothing to unpack and a policy step over it raised a ValueError. Fall back to rebuilding the chunks as they are when no choice ever appeared. --- .../openai/chat/guardrail_translation/handler.py | 3 +++ .../test_openai_guardrail_handler.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3d45feb0eab..d79d2ba5e3c 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -856,6 +856,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for index in choice_indices ) + if not rebuilt_by_index: + whole: Final = stream_chunk_builder(chunks=list(responses_so_far), logging_obj=litellm_logging_obj) + return cast(ModelResponse, whole) if whole is not None else ModelResponse(choices=[]) (_, base_response), *_ = rebuilt_by_index stitched_choices: Final = [ rebuilt.choices[0].model_copy(update=MappingProxyType({"index": index})) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index f139e2c90fc..0e4ca539165 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1669,6 +1669,22 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_a_buffered_stream_that_carried_no_choices_is_not_an_error(self) -> None: + """A stream can end carrying only usage, and a policy step still runs over it.""" + from litellm.types.utils import ModelResponseStream + + chunks: Final = [ModelResponseStream(choices=[], usage={"prompt_tokens": 3, "completion_tokens": 0, + "total_tokens": 3})] + result: Final = await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + request_data={}, + deliver_ended_stream_rewrites=True, + ) + assert result is chunks + @pytest.mark.asyncio async def test_a_round_with_nothing_to_judge_costs_one_call_at_n2(self) -> None: """Per-choice rounds buy tool-call context. While no call is complete there is none to buy,