From 95348ba610fcd21b742c64786d1fa0eed7d4b9f4 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:36:57 +0200 Subject: [PATCH 01/15] fix(guardrails): isolate streamed completion choices during inspection --- .../guardrail_translation/base_translation.py | 10 +- .../chat/guardrail_translation/handler.py | 220 +++++++++++------- .../test_openai_guardrail_handler.py | 70 ++++-- 3 files changed, 190 insertions(+), 110 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 88c4dbfdce6..6280e636700 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional @@ -31,8 +31,12 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: dict[int, str] = field(default_factory=dict) - holdback_per_choice: dict[int, int] = field(default_factory=dict) + mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) + holdback_per_choice: Mapping[int, int] = field(default_factory=dict) + + def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: + self.mutated_text_per_choice = texts + self.holdback_per_choice = holdback @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a12f5f164e4..d18fe36e3df 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -18,9 +18,11 @@ import json import time import uuid from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -46,7 +48,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -168,8 +170,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) + guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ()) guardrailed_tools: Final = guardrailed_inputs.get("tools") if guardrailed_tools is not None: data["tools"] = ( @@ -258,12 +260,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: + for tool in data.get("tools") or (): if isinstance(tool, dict) and tool.get("type") == "function": fn = tool.get("function") if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - for fn in data.get("functions") or []: + for fn in data.get("functions") or (): if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) return names @@ -387,7 +389,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, ) -> ModelResponse: """ Process output response by applying guardrails to text content. @@ -438,9 +440,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # create a standalone dict (SDK / direct-call path). if request_data is None: request_data = {"response": response} - else: - if "response" not in request_data: - request_data["response"] = response + request_data.setdefault("response", response) self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) @@ -460,7 +460,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) @@ -494,7 +494,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, stream_transform_sink: StreamTransformSink | None = None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: @@ -550,14 +550,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path) unless ``deliver_ended_stream_rewrites`` opts the ended-stream branch in.""" - has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) + has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far) if has_stream_ended: await self._process_ended_stream( @@ -623,7 +623,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) # Step 4: Apply guardrailed text back to all streaming chunks # For each choice, replace the combined text across all chunks @@ -656,13 +656,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation): model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) pre_guardrail_texts: Final = self._string_choice_contents(model_response) pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) - await self.process_output_response( - response=model_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=request_data, + inspection_responses: Final = ( + tuple( + model_response.model_copy(update=MappingProxyType({"choices": [choice]})) + for choice in model_response.choices + ) + if pre_guardrail_tool_calls and len(model_response.choices) > 1 + else (model_response,) ) + inspection_request_data: Final = request_data if request_data is not None else {} + try: + for inspection_response in inspection_responses: + inspection_request_data["response"] = inspection_response + inspection_request_data["responses"] = ( + [ + self._narrowed_to_choice(chunk, inspection_response.choices[0].index) + for chunk in responses_so_far + ] + if len(inspection_response.choices) == 1 + else responses_so_far + ) + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + finally: + inspection_request_data["response"] = model_response + inspection_request_data["responses"] = responses_so_far if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -763,7 +786,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, sink: StreamTransformSink, ) -> None: """Run the guardrail over the raw accumulated text and report the @@ -775,8 +798,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far) if not raw_by_index: - sink.mutated_text_per_choice = {} - sink.holdback_per_choice = {} + sink.record(MappingProxyType({}), MappingProxyType({})) return # Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits @@ -788,18 +810,72 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices: Final = sorted(raw_by_index.keys()) texts_to_check: Final = [raw_by_index[i] for i in indices] - if request_data is None: - request_data = {"responses": responses_so_far} - elif "responses" not in request_data: - request_data["responses"] = responses_so_far - self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) + resolved_request_data: Final[dict[str, object]] = ( + request_data if request_data is not None else {"responses": responses_so_far} + ) + if "responses" not in resolved_request_data: + resolved_request_data["responses"] = responses_so_far + self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) + if self._streamed_tool_call_fingerprints(responses_so_far): + assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + resolved_request_data["response"] = assembled + if len(assembled.choices) > 1: + choice_rounds: Final = tuple( + ( + choice, + StreamTransformSink(), + [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], + ) + for choice in assembled.choices + ) + try: + for choice, choice_sink, choice_chunks in choice_rounds: + resolved_request_data["response"] = assembled.model_copy( + update=MappingProxyType({"choices": [choice]}) + ) + resolved_request_data["responses"] = choice_chunks + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + finally: + resolved_request_data["response"] = assembled + resolved_request_data["responses"] = responses_so_far + merged_texts: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + merged_holdback: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + sink.record(merged_texts, merged_holdback) + return + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) + ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, - request_data=request_data, + request_data=resolved_request_data, input_type="response", logging_obj=litellm_logging_obj, ) @@ -819,13 +895,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): len(texts_to_check), ) - holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or [] - sink.mutated_text_per_choice = { - idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts) - } - sink.holdback_per_choice = { - indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) - } + holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () + transformed_texts: Final = MappingProxyType( + {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} + ) + transformed_holdback: Final = MappingProxyType( + {indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)} + ) + sink.record(transformed_texts, transformed_holdback) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) @@ -1145,20 +1222,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _function_tool_call_fragments( responses_so_far: Sequence["ModelResponseStream"], ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: - """Group the stream's function tool-call fragments by their tool-call index, in - the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping - only the indices the builder keeps (an id and a name somewhere in the stream).""" fragments: Final = tuple( - tool_call + (choice.index, tool_call) for response in responses_so_far for choice in response.choices for tool_call in choice.delta.tool_calls or () if isinstance(tool_call, ChatCompletionDeltaToolCall) ) - identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) - named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id) + named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name) return tuple( - tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key) + for key in sorted(identified & named) ) def _write_ended_stream_tool_call_rewrites( @@ -1168,28 +1243,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], guardrail_name: str, ) -> None: - """Write ended-stream guardrail tool-call rewrites back across the buffered - chunks: the rewritten name and full arguments land in the tool call's first - fragment and the arguments of its later fragments are blanked, mirroring the - text write-back. A rewrite on a stream carrying more than one distinct choice - index, or whose fragments do not line up with the rebuilt tool calls, is - reported as undeliverable, so the pipeline executor discards it and releases - the original chunks.""" post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - stream_choice_indices: Final = frozenset( - choice.index for response in responses_so_far for choice in response.choices - ) fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) - if len(stream_choice_indices) != 1: - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite( - guardrail_name, - f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written " - "back on single-choice streams", - ) if len(fragments_by_tool_call) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite @@ -1212,7 +1269,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], - guardrailed_texts: list[str], + guardrailed_texts: Sequence[str], task_mappings: list[tuple[int, int | None]], ) -> None: """ @@ -1230,12 +1287,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Override this method to customize how responses are applied to streaming responses. """ # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx) - guardrail_map: Final[dict[tuple[int, int | None], str]] = {} - for task_idx, guardrail_response in enumerate(guardrailed_texts): - mapping = task_mappings[task_idx] - choice_idx = cast(int, mapping[0]) - content_idx_optional = cast(int | None, mapping[1]) - guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response + guardrail_map: Final = MappingProxyType( + {task_mappings[index]: text for index, text in enumerate(guardrailed_texts)} + ) # Track which choices we've already set the guardrailed text for # Key: (choice_idx, content_idx), Value: boolean (True if already set) @@ -1245,34 +1299,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # choice by its index field: on n>1 streams a chunk usually carries one # choice at list position 0 whose index names the logical choice. for response in responses: - for choice in response.choices: - if isinstance(choice, litellm.StreamingChoices): - content = choice.delta.content - elif isinstance(choice, litellm.Choices): - content = choice.message.content - else: - continue - + for choice, message in ( + (choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message) + for choice in response.choices + if isinstance(choice, (StreamingChoices, Choices)) + ): + content = message.content if content is None: continue if isinstance(content, str): - # String content str_key: tuple[int, int | None] = (choice.index, None) if str_key in guardrail_map: - if str_key not in already_set: - # First chunk - set the complete guardrailed text - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = guardrail_map[str_key] - elif isinstance(choice, litellm.Choices): - choice.message.content = guardrail_map[str_key] - already_set[str_key] = True - else: - # Subsequent chunks - clear the content - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = "" - elif isinstance(choice, litellm.Choices): - choice.message.content = "" + message.content = guardrail_map[str_key] if str_key not in already_set else "" + already_set[str_key] = True elif isinstance(content, list): # List content - handle each content item diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..bffe6e4c123 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Literal, Optional +from typing import Any, Final, Literal, Optional import pytest @@ -1395,29 +1395,65 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert [ - (tool_call["id"], tool_call["function"]["arguments"]) - for tool_call in guardrail.seen_inputs[-1]["tool_calls"] - ] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')] + [(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]] + for inputs in guardrail.seen_inputs + ] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]] @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + @pytest.mark.parametrize("transform", [False, True]) + async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None: + from fastapi import HTTPException + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - handler = OpenAIChatCompletionsHandler() - chunks = self._two_choice_tool_call_stream_chunks() + class ContextGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index) + if response.choices[0].message.content == "forbidden": + raise HTTPException(status_code=400, detail="later choice blocked") + return inputs - with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised: - await handler.process_output_streaming_response( - responses_so_far=chunks, - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, - deliver_ended_stream_rewrites=True, + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("allowed", "forbidden"))])] + request_data: Final = {"metadata": {"trace": "retained"}} + with pytest.raises(HTTPException, match="later choice blocked"): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink() if transform else None, ) + assert request_data["visited_choices"] == (0, 1) + assert request_data["metadata"]["trace"] == "retained" + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) - assert raised.value.guardrail_name == "test" - assert raised.value.reason == ( - "the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams" + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: + handler: Final = OpenAIChatCompletionsHandler() + chunks: Final = self._two_choice_tool_call_stream_chunks() + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, ) + arguments: Final = tuple( + "".join(call.function.arguments for chunk in chunks for choice in chunk.choices + if choice.index == index for call in choice.delta.tool_calls or ()) + for index in range(2) + ) + assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}') @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): From c99cc59ddaeb29f0f30ec0693b9314e67b315e6c Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:43:10 +0200 Subject: [PATCH 02/15] fix(guardrails): preserve the existing stream sink value types --- .../base_llm/guardrail_translation/base_translation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 6280e636700..f3dd67a2bb9 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -31,12 +31,12 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) - holdback_per_choice: Mapping[int, int] = field(default_factory=dict) + mutated_text_per_choice: dict[int, str] = field(default_factory=dict) + holdback_per_choice: dict[int, int] = field(default_factory=dict) def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: - self.mutated_text_per_choice = texts - self.holdback_per_choice = holdback + self.mutated_text_per_choice = dict(texts) + self.holdback_per_choice = dict(holdback) @dataclass(frozen=True, slots=True) From b2b61d21999dca4fa086d7b19f5fd41f5895b990 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:00:45 +0200 Subject: [PATCH 03/15] test(guardrails): verify disconnect inspection retains tool context --- .../unified_guardrails/test_unified_guardrail.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..9a4fa3ba91b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2706,7 +2706,12 @@ class TestStreamingClientDisconnectScan: assert [call.function.name for call in released[0].choices[0].delta.tool_calls] == ["get_weather"] assert guardrail.received_texts == [["BLOCKME"]], guardrail.received_texts - assert guardrail.received_tool_calls == [], guardrail.received_tool_calls + assert guardrail.received_tool_calls == [[{ + "id": "call_1", + "type": "function", + "index": 0, + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }]] @pytest.mark.asyncio async def test_closing_after_a_released_tool_call_under_incremental_diff_scans_no_held_back_text(self): From 37d6a1cacd3493ffdf50430a2896672776fa105b Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:03:34 +0200 Subject: [PATCH 04/15] refactor(guardrails): type the shared constructor extension fields --- litellm/integrations/custom_guardrail.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 45b20211ac5..583e9108af7 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,8 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, - **kwargs, - ): + **kwargs: object, + ) -> None: """ Initialize the CustomGuardrail class @@ -303,7 +303,7 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages - stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config + stream_scope_arg: Final[object] = kwargs.pop("stream_scope", None) self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None self.logging_only_continue_on_input_failure = False From a07ec080365c633968a15c0198f6afac6c4683d1 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:09:17 +0200 Subject: [PATCH 05/15] fix(guardrails): forward typed message logging flags to the logger --- litellm/integrations/custom_guardrail.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 583e9108af7..f7457dd4ff9 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,6 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, + turn_off_message_logging: bool = False, + message_logging: bool = True, **kwargs: object, ) -> None: """ @@ -323,7 +325,11 @@ class CustomGuardrail(CustomLogger): "(default) to enforce validation and fail at startup.", validation_error, ) - super().__init__(**kwargs) + super().__init__( + turn_off_message_logging=turn_off_message_logging, + message_logging=message_logging, + **kwargs, + ) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: """Return a custom violation message if template is configured.""" From fa10963e3d3fd03020c07a35d9c5d09bae3a7b80 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:11:45 +0200 Subject: [PATCH 06/15] refactor(guardrails): keep legacy constructor typing outside choice inspection --- litellm/integrations/custom_guardrail.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index f7457dd4ff9..45b20211ac5 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,10 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, - turn_off_message_logging: bool = False, - message_logging: bool = True, - **kwargs: object, - ) -> None: + **kwargs, + ): """ Initialize the CustomGuardrail class @@ -305,7 +303,7 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages - stream_scope_arg: Final[object] = kwargs.pop("stream_scope", None) + stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None self.logging_only_continue_on_input_failure = False @@ -325,11 +323,7 @@ class CustomGuardrail(CustomLogger): "(default) to enforce validation and fail at startup.", validation_error, ) - super().__init__( - turn_off_message_logging=turn_off_message_logging, - message_logging=message_logging, - **kwargs, - ) + super().__init__(**kwargs) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: """Return a custom violation message if template is configured.""" From 7254b3ad5ee17b6270a474ee2f8b84833be4cbfd Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:02:25 +0100 Subject: [PATCH 07/15] fix(guardrails): inspect every choice before raising an undeliverable rewrite --- .../chat/guardrail_translation/handler.py | 75 ++++++++++++---- .../test_openai_guardrail_handler.py | 89 ++++++++++++++++++- 2 files changed, 143 insertions(+), 21 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a4a3956df94..9ad42b18c70 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -669,7 +669,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if pre_guardrail_tool_calls and len(model_response.choices) > 1 else (model_response,) ) + # Local import avoids a module-level cyclic import with the policy engine. + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + inspection_request_data: Final = request_data if request_data is not None else {} + missing: Final = object() + prior_response: Final = inspection_request_data.get("response", missing) + prior_responses: Final = inspection_request_data.get("responses", missing) + # An undeliverable rewrite on one choice must not end the loop: the caller releases the + # original chunks when it catches this, so leaving later choices uninspected would let a + # request with n > 1 pair a redactable choice with a prohibited one and have the whole + # response delivered unchecked. Inspect every choice, then raise the first failure. + undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response @@ -681,16 +692,31 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if len(inspection_response.choices) == 1 else responses_so_far ) - await self.process_output_response( - response=inspection_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=inspection_request_data, - ) + try: + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + except UndeliverableStreamRewrite as exc: + if undeliverable is None: + undeliverable = exc finally: - inspection_request_data["response"] = model_response - inspection_request_data["responses"] = responses_so_far + # Restore the caller's keys rather than overwriting them: the inspection loop owns + # these entries only for its own duration, and "responses" is not a key the + # non-streaming path sets at all. + if prior_response is missing: + inspection_request_data.pop("response", None) + else: + inspection_request_data["response"] = prior_response + if prior_responses is missing: + inspection_request_data.pop("responses", None) + else: + inspection_request_data["responses"] = prior_responses + if undeliverable is not None: + raise undeliverable if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -835,23 +861,38 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for choice in assembled.choices ) + # Local import avoids a module-level cyclic import with the policy engine. + from litellm.proxy.policy_engine.pipeline_executor import ( + UndeliverableStreamRewrite, + ) + + # Same rule as the ended-stream inspection loop: an undeliverable rewrite on one + # choice must not leave the rest uninspected, because the caller releases the + # original chunks when it catches this. + transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: resolved_request_data["response"] = assembled.model_copy( update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks - await self._process_streaming_transform( - responses_so_far=choice_chunks, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=resolved_request_data, - sink=choice_sink, - ) + try: + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + except UndeliverableStreamRewrite as exc: + if transform_undeliverable is None: + transform_undeliverable = exc finally: resolved_request_data["response"] = assembled resolved_request_data["responses"] = responses_so_far + if transform_undeliverable is not None: + raise transform_undeliverable merged_texts: Final = MappingProxyType( dict( chain.from_iterable( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bffe6e4c123..f21e22545f5 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1433,10 +1433,91 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" - assert request_data["responses"] is chunks - restored: Final = request_data["response"] - assert isinstance(restored, ModelResponse) - assert tuple(choice.index for choice in restored.choices) == (0, 1) + if transform: + # The transform path seeds "response" and "responses" up front as the context the + # guardrail reads, so they are its own values rather than leftovers. + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) + else: + # The ended-stream inspection loop owns these keys only while it runs; neither was in + # request_data beforehand, and the non-streaming path never sets "responses" at all. + assert "response" not in request_data + assert "responses" not in request_data + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None: + """An undeliverable rewrite on one choice must not leave later choices uninspected. + + The caller releases the original chunks when it catches UndeliverableStreamRewrite, so + bailing out of the loop would let a request with n > 1 pair a redactable choice with a + prohibited one and have the whole response delivered unchecked. + """ + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + ) + assert request_data["visited_choices"] == (0, 1) + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None: + """The transform path runs its own per-choice loop and needs the same rule.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class UndeliverableOnFirstChoice(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + index: Final = response.choices[0].index + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index) + if index == 0: + raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch") + return inputs + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("redacted", "prohibited"))])] + request_data: Final = {} + with pytest.raises(UndeliverableStreamRewrite): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink(), + ) + assert request_data["visited_choices"] == (0, 1) @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: From c6e386cefd25122d2eb134c3fe8a0169856a1f9e Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:13:25 +0100 Subject: [PATCH 08/15] test(guardrails): cover the buffered ended-stream path and per-choice holdback --- .../test_openai_guardrail_handler.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index f21e22545f5..bc8ffe07b4f 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1519,6 +1519,73 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert request_data["visited_choices"] == (0, 1) + @pytest.mark.asyncio + async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None: + """A buffered stream has no finish_reason yet, so the ended path must key off the opt-in. + + Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered + caller on the in-flight path, where rewrites are never written back. + """ + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class PathProbe(CustomGuardrail): + saw_assembled_response: bool = False + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + # Only the ended-stream inspection loop hands the guardrail an assembled + # "response"; the in-flight branch passes raw chunks under "responses" alone. + type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse) + return inputs + + chunks: Final = [ + ModelResponseStream( + choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)] + ) + ] + assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=PathProbe(guardrail_name="probe"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, + ) + assert PathProbe.saw_assembled_response is True + + @pytest.mark.asyncio + async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None: + """The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks. + + Reporting an empty holdback here would release text the guardrail asked to withhold. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class HoldingGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + texts: Final = inputs.get("texts") or () + return {**inputs, "stream_holdback_chars": [len(text) for text in texts]} + + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("alpha", "bravo-long"))])] + sink: Final = StreamTransformSink() + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"), + litellm_logging_obj=None, + stream_transform_sink=sink, + ) + assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")} + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From fd3ad67c474b162200e757a0fcdb01926035e4c4 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:22:00 +0100 Subject: [PATCH 09/15] fix(guardrails): send tool calls to the guardrail only once their arguments are complete --- .../chat/guardrail_translation/handler.py | 19 ++++--- .../test_openai_guardrail_handler.py | 49 +++++++++++++++++++ 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 9ad42b18c70..ef9c1eb34ab 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -909,14 +909,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) sink.record(merged_texts, merged_holdback) return - tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) - inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( - tuple( - MappingProxyType({"index": index, **converted}) - for index, tool_call in enumerate(tool_calls) - if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which + # main never handed to a guardrail: one that parses them either errors or judges + # arguments that do not exist yet. Send tool calls only once the stream has finished and + # the arguments are complete; the text of those rounds is still inspected above. + if self._first_choice_has_finished(responses_so_far): + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) ) - ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bc8ffe07b4f..a503bbf386e 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1586,6 +1586,55 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")} + @pytest.mark.asyncio + async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None: + """Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent. + + Sending them would hand the guardrail something like `{"ci`, which main never did. + """ + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(inputs.get("tool_calls")) + return inputs + + def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream: + # The text keeps the guardrail in play for the partial round; without any text the + # transform path has nothing to inspect and never calls it at all. + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + }]), finish_reason=finish_reason, + )]) + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + partial: Final = [chunk("looking", '{"ci', None)] + await handler.process_output_streaming_response( + responses_so_far=partial, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")] + await handler.process_output_streaming_response( + responses_so_far=complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From ff5644b3d873bfce95e6ff300432631d135a2005 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 12:32:06 +0100 Subject: [PATCH 10/15] perf(guardrails): rebuild the stream only when a round consumes it --- .../chat/guardrail_translation/handler.py | 22 +++++++-- .../test_openai_guardrail_handler.py | 47 +++++++++++++++++++ 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index ef9c1eb34ab..a43c6b193b6 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -850,9 +850,23 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - resolved_request_data["response"] = assembled - if len(assembled.choices) > 1: + # Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is + # quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild. + # main rebuilds only on the ended path, so rebuild here only when this round consumes the + # assembled response -- to isolate choices, or to hand over complete tool calls once the + # stream has finished. A single-choice round in flight needs neither. + stream_ended: Final = self._first_choice_has_finished(responses_so_far) + streamed_choice_indices: Final = frozenset( + choice.index for chunk in responses_so_far for choice in chunk.choices + ) + assembled: Final = ( + None + if len(streamed_choice_indices) == 1 and not stream_ended + else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + ) + if assembled is not None: + resolved_request_data["response"] = assembled + if assembled is not None and len(assembled.choices) > 1: choice_rounds: Final = tuple( ( choice, @@ -913,7 +927,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # main never handed to a guardrail: one that parses them either errors or judges # arguments that do not exist yet. Send tool calls only once the stream has finished and # the arguments are complete; the text of those rounds is still inspected above. - if self._first_choice_has_finished(responses_so_far): + if assembled is not None and stream_ended: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( tuple( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index a503bbf386e..137383c3483 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1635,6 +1635,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] + @pytest.mark.asyncio + async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None: + """Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length. + + main rebuilds only on the ended path; an in-flight single-choice round consumes nothing + from the assembled response, so it must not pay for one. + """ + from unittest.mock import patch + + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def chunk(text: str, finish_reason: str | None) -> ModelResponseStream: + return ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": "call_0", "type": "function", + "function": {"name": "weather", "arguments": '{"c'}, + }]), finish_reason=finish_reason, + )]) + + handler: Final = OpenAIChatCompletionsHandler() + rebuilds: Final[list[int]] = [] + original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice + + def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy + rebuilds.append(len(responses_so_far)) + return original(responses_so_far, litellm_logging_obj) + + # The real attribute is a staticmethod, so the spy has to be one too: a plain function + # would be bound and receive the handler as its first argument. + with patch.object( + OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild) + ): + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None)], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [] + + await handler.process_output_streaming_response( + responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")], + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + assert rebuilds == [2] + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From f2342b832d051a4800c03e737981bb932bd8d2f3 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 13:09:43 +0100 Subject: [PATCH 11/15] feat(guardrails): label per-choice log entries and gate tool calls on complete arguments --- litellm/integrations/custom_guardrail.py | 3 ++ .../chat/guardrail_translation/handler.py | 47 +++++++++++++++++-- litellm/types/utils.py | 3 ++ .../test_openai_guardrail_handler.py | 30 ++++++++++++ 4 files changed, 79 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 9e2dc902d55..7b1eaca13af 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -1414,6 +1414,9 @@ class CustomGuardrail(CustomLogger): end_time=end_time, duration=duration, masked_entity_count=masked_entity_count, + # Set by callers that inspect one choice at a time, so the per-choice entries of an + # n > 1 response can be told apart. + choice_index=request_data.get("guardrail_choice_index"), **(tracing_detail or {}), ) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a43c6b193b6..be698093fec 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -676,6 +676,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): missing: Final = object() prior_response: Final = inspection_request_data.get("response", missing) prior_responses: Final = inspection_request_data.get("responses", missing) + prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) # An undeliverable rewrite on one choice must not end the loop: the caller releases the # original chunks when it catches this, so leaving later choices uninspected would let a # request with n > 1 pair a redactable choice with a prohibited one and have the whole @@ -684,6 +685,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response + # Labels this round's guardrail log entry and span with the choice it inspected. + inspection_request_data["guardrail_choice_index"] = ( + inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None + ) inspection_request_data["responses"] = ( [ self._narrowed_to_choice(chunk, inspection_response.choices[0].index) @@ -715,6 +720,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inspection_request_data.pop("responses", None) else: inspection_request_data["responses"] = prior_responses + if prior_choice_index is missing: + inspection_request_data.pop("guardrail_choice_index", None) + else: + inspection_request_data["guardrail_choice_index"] = prior_choice_index if undeliverable is not None: raise undeliverable if not deliver_ended_stream_rewrites: @@ -731,6 +740,31 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + @staticmethod + def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: + """Whether every streamed tool call now carries arguments that parse as JSON. + + Streamed argument fragments are only valid JSON once the last one has arrived, so this is + the completeness test: it holds back `{"ci` and lets `{"city": "Paris"}` through whether + the stream ended normally or was cut short by a disconnect. It reads the raw chunks so the + answer is available before deciding whether to rebuild. + """ + fragments: Final[dict[tuple[int, int], list[str]]] = {} # mutable-ok: local accumulator + for chunk in responses_so_far: + for choice in chunk.choices: + for call in getattr(choice.delta, "tool_calls", None) or (): + arguments = getattr(getattr(call, "function", None), "arguments", None) + if isinstance(arguments, str) and arguments: + fragments.setdefault((choice.index, getattr(call, "index", 0)), []).append(arguments) + if not fragments: + return False + for parts in fragments.values(): + try: + json.loads("".join(parts)) + except ValueError: + return False + return True + @staticmethod def _rebuild_ended_stream_per_choice( responses_so_far: Sequence["ModelResponseStream"], @@ -859,9 +893,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): streamed_choice_indices: Final = frozenset( choice.index for chunk in responses_so_far for choice in chunk.choices ) + tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) assembled: Final = ( None - if len(streamed_choice_indices) == 1 and not stream_ended + if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) if assembled is not None: @@ -890,6 +925,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): update=MappingProxyType({"choices": [choice]}) ) resolved_request_data["responses"] = choice_chunks + resolved_request_data["guardrail_choice_index"] = choice.index try: await self._process_streaming_transform( responses_so_far=choice_chunks, @@ -905,6 +941,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): finally: resolved_request_data["response"] = assembled resolved_request_data["responses"] = responses_so_far + resolved_request_data.pop("guardrail_choice_index", None) if transform_undeliverable is not None: raise transform_undeliverable merged_texts: Final = MappingProxyType( @@ -925,9 +962,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which # main never handed to a guardrail: one that parses them either errors or judges - # arguments that do not exist yet. Send tool calls only once the stream has finished and - # the arguments are complete; the text of those rounds is still inspected above. - if assembled is not None and stream_ended: + # arguments that do not exist yet. Completeness is what matters, not the finish reason: + # a stream cut short by a client disconnect carries whole arguments and no finish + # reason, and that call still has to be inspected. Send the calls once every one of + # them parses; the text of the earlier rounds is still inspected above. + if assembled is not None and tool_arguments_complete: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( tuple( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..c6eeb41a9d1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3551,6 +3551,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False): } """ + choice_index: ReadOnly[int | None] + """Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice.""" + guardrail_id: str | None """Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'""" diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 137383c3483..963b7dde07b 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1682,6 +1682,36 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert rebuilds == [2] + @pytest.mark.asyncio + async def test_per_choice_inspection_labels_its_log_entries_with_the_choice_index(self) -> None: + """Without a label the per-choice entries of an n > 1 response cannot be told apart.""" + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class IndexRecorder(CustomGuardrail): + seen: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen.append(request_data.get("guardrail_choice_index")) + return inputs + + IndexRecorder.seen = [] + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("first", "second"))])] + request_data: Final = {} + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=IndexRecorder(guardrail_name="indexed"), + request_data=request_data, deliver_ended_stream_rewrites=True, + ) + assert IndexRecorder.seen == [0, 1] + assert "guardrail_choice_index" not in request_data + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler() From 5a2e78e55052c1ce8ef40d5bfebe86c65d46e4f5 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 18:33:51 +0100 Subject: [PATCH 12/15] fix(types): include choice_index in the guardrail audit fields --- litellm/types/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c6eeb41a9d1..d27c4bb62a6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3511,6 +3511,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset( "end_time", "duration", "masked_entity_count", + "choice_index", "guardrail_id", "policy_template", "detection_method", From 55545c693fe5f6183d74b1ca21c3eb00ce57a868 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 28 Oct 2026 23:25:18 +0100 Subject: [PATCH 13/15] fix(guardrails): treat tool calls with no arguments yet as incomplete --- .../chat/guardrail_translation/handler.py | 81 ++++++-------- .../test_openai_guardrail_handler.py | 105 ++++++------------ 2 files changed, 71 insertions(+), 115 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index be698093fec..18c8b5aca9e 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -669,7 +669,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if pre_guardrail_tool_calls and len(model_response.choices) > 1 else (model_response,) ) - # Local import avoids a module-level cyclic import with the policy engine. from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite inspection_request_data: Final = request_data if request_data is not None else {} @@ -677,15 +676,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): prior_response: Final = inspection_request_data.get("response", missing) prior_responses: Final = inspection_request_data.get("responses", missing) prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing) - # An undeliverable rewrite on one choice must not end the loop: the caller releases the - # original chunks when it catches this, so leaving later choices uninspected would let a - # request with n > 1 pair a redactable choice with a prohibited one and have the whole - # response delivered unchecked. Inspect every choice, then raise the first failure. undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for inspection_response in inspection_responses: inspection_request_data["response"] = inspection_response - # Labels this round's guardrail log entry and span with the choice it inspected. inspection_request_data["guardrail_choice_index"] = ( inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None ) @@ -709,9 +703,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): if undeliverable is None: undeliverable = exc finally: - # Restore the caller's keys rather than overwriting them: the inspection loop owns - # these entries only for its own duration, and "responses" is not a key the - # non-streaming path sets at all. if prior_response is missing: inspection_request_data.pop("response", None) else: @@ -740,30 +731,49 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_name=guardrail_to_apply.guardrail_name or "unknown", ) + @staticmethod + def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]: + for chunk in responses_so_far: + for choice in chunk.choices: + yield choice.index + + @staticmethod + def _streamed_argument_fragments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> Iterator[tuple[tuple[int, int], str]]: + for chunk in responses_so_far: + for choice in chunk.choices: + for call in choice.delta.tool_calls or (): + arguments = getattr(getattr(call, "function", None), "arguments", None) + yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else "" + + @staticmethod + def _parses_as_json(arguments: str) -> bool: + if not arguments: + return False + try: + json.loads(arguments) + except ValueError: + return False + return True + @staticmethod def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: """Whether every streamed tool call now carries arguments that parse as JSON. - Streamed argument fragments are only valid JSON once the last one has arrived, so this is - the completeness test: it holds back `{"ci` and lets `{"city": "Paris"}` through whether - the stream ended normally or was cut short by a disconnect. It reads the raw chunks so the - answer is available before deciding whether to rebuild. + A call that has announced itself but emitted no argument text yet is still in flight, so a + parallel call finishing first does not make the whole set complete. """ - fragments: Final[dict[tuple[int, int], list[str]]] = {} # mutable-ok: local accumulator - for chunk in responses_so_far: - for choice in chunk.choices: - for call in getattr(choice.delta, "tool_calls", None) or (): - arguments = getattr(getattr(call, "function", None), "arguments", None) - if isinstance(arguments, str) and arguments: - fragments.setdefault((choice.index, getattr(call, "index", 0)), []).append(arguments) + fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) if not fragments: return False - for parts in fragments.values(): - try: - json.loads("".join(parts)) - except ValueError: - return False - return True + joined: Final = MappingProxyType( + { + call: "".join(text for other, text in fragments if other == call) + for call in frozenset(call for call, _ in fragments) + } + ) + return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) @staticmethod def _rebuild_ended_stream_per_choice( @@ -884,15 +894,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) if self._streamed_tool_call_fingerprints(responses_so_far): - # Rebuilding folds every chunk accumulated so far, so doing it on every sampled round is - # quadratic in stream length: a 400-chunk stream costs ~44x one end-of-stream rebuild. - # main rebuilds only on the ended path, so rebuild here only when this round consumes the - # assembled response -- to isolate choices, or to hand over complete tool calls once the - # stream has finished. A single-choice round in flight needs neither. stream_ended: Final = self._first_choice_has_finished(responses_so_far) - streamed_choice_indices: Final = frozenset( - choice.index for chunk in responses_so_far for choice in chunk.choices - ) + streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far)) tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far) assembled: Final = ( None @@ -910,14 +913,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) for choice in assembled.choices ) - # Local import avoids a module-level cyclic import with the policy engine. from litellm.proxy.policy_engine.pipeline_executor import ( UndeliverableStreamRewrite, ) - # Same rule as the ended-stream inspection loop: an undeliverable rewrite on one - # choice must not leave the rest uninspected, because the caller releases the - # original chunks when it catches this. transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure try: for choice, choice_sink, choice_chunks in choice_rounds: @@ -960,12 +959,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) sink.record(merged_texts, merged_holdback) return - # Mid-round the assembled arguments are a truncated JSON fragment such as `{"ci`, which - # main never handed to a guardrail: one that parses them either errors or judges - # arguments that do not exist yet. Completeness is what matters, not the finish reason: - # a stream cut short by a client disconnect carries whole arguments and no finish - # reason, and that call still has to be inspected. Send the calls once every one of - # them parses; the text of the earlier rounds is still inspected above. if assembled is not None and tool_arguments_complete: tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 963b7dde07b..0a0bccceb71 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1434,15 +1434,11 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert request_data["visited_choices"] == (0, 1) assert request_data["metadata"]["trace"] == "retained" if transform: - # The transform path seeds "response" and "responses" up front as the context the - # guardrail reads, so they are its own values rather than leftovers. assert request_data["responses"] is chunks restored: Final = request_data["response"] assert isinstance(restored, ModelResponse) assert tuple(choice.index for choice in restored.choices) == (0, 1) else: - # The ended-stream inspection loop owns these keys only while it runs; neither was in - # request_data beforehand, and the non-streaming path never sets "responses" at all. assert "response" not in request_data assert "responses" not in request_data @@ -1535,8 +1531,6 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], input_type: Literal["request", "response"], logging_obj: object = None, ) -> GenericGuardrailAPIInputs: - # Only the ended-stream inspection loop hands the guardrail an assembled - # "response"; the in-flight branch passes raw chunks under "responses" alone. type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse) return inputs @@ -1606,8 +1600,6 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: return inputs def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream: - # The text keeps the guardrail in play for the partial round; without any text the - # transform path has nothing to inspect and never calls it at all. return ModelResponseStream(choices=[StreamingChoices( index=0, delta=Delta(content=text, tool_calls=[{ "index": 0, "id": "call_0", "type": "function", @@ -1636,81 +1628,52 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}'] @pytest.mark.asyncio - async def test_diff_mode_does_not_rebuild_the_stream_on_in_flight_rounds(self) -> None: - """Rebuilding folds every chunk so far, so a per-round rebuild is quadratic in length. - - main rebuilds only on the ended path; an in-flight single-choice round consumes nothing - from the assembled response, so it must not pay for one. - """ - from unittest.mock import patch - + async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None: from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - def chunk(text: str, finish_reason: str | None) -> ModelResponseStream: - return ModelResponseStream(choices=[StreamingChoices( - index=0, delta=Delta(content=text, tool_calls=[{ - "index": 0, "id": "call_0", "type": "function", - "function": {"name": "weather", "arguments": '{"c'}, - }]), finish_reason=finish_reason, - )]) - - handler: Final = OpenAIChatCompletionsHandler() - rebuilds: Final[list[int]] = [] - original: Final = OpenAIChatCompletionsHandler._rebuild_ended_stream_per_choice - - def counting_rebuild(responses_so_far, litellm_logging_obj): # type: ignore[no-untyped-def] # test spy - rebuilds.append(len(responses_so_far)) - return original(responses_so_far, litellm_logging_obj) - - # The real attribute is a staticmethod, so the spy has to be one too: a plain function - # would be bound and receive the handler as its first argument. - with patch.object( - OpenAIChatCompletionsHandler, "_rebuild_ended_stream_per_choice", staticmethod(counting_rebuild) - ): - await handler.process_output_streaming_response( - responses_so_far=[chunk("looking", None)], - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), - ) - assert rebuilds == [] - - await handler.process_output_streaming_response( - responses_so_far=[chunk("looking", None), chunk(" it up", "tool_calls")], - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), - ) - assert rebuilds == [2] - - @pytest.mark.asyncio - async def test_per_choice_inspection_labels_its_log_entries_with_the_choice_index(self) -> None: - """Without a label the per-choice entries of an n > 1 response cannot be told apart.""" - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - - class IndexRecorder(CustomGuardrail): + class RecordingGuardrail(CustomGuardrail): seen: list[object] = [] async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], input_type: Literal["request", "response"], logging_obj: object = None, ) -> GenericGuardrailAPIInputs: - type(self).seen.append(request_data.get("guardrail_choice_index")) + type(self).seen.append(inputs.get("tool_calls")) return inputs - IndexRecorder.seen = [] - chunks: Final = [ModelResponseStream(choices=[StreamingChoices( - index=index, delta=Delta(content=text, tool_calls=[{ - "index": 0, "id": f"call_{index}", "type": "function", - "function": {"name": "contact", "arguments": "{}"}, - }]), finish_reason="tool_calls", - ) for index, text in enumerate(("first", "second"))])] - request_data: Final = {} - await OpenAIChatCompletionsHandler().process_output_streaming_response( - responses_so_far=chunks, guardrail_to_apply=IndexRecorder(guardrail_name="indexed"), - request_data=request_data, deliver_ended_stream_rewrites=True, + def call(index: int, arguments: str) -> dict[str, object]: + return { + "index": index, "id": f"call_{index}", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + } + + RecordingGuardrail.seen = [] + handler: Final = OpenAIChatCompletionsHandler() + guardrail: Final = RecordingGuardrail(guardrail_name="recording") + one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]), + finish_reason=None, + )])] + await handler.process_output_streaming_response( + responses_so_far=one_still_empty, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), ) - assert IndexRecorder.seen == [0, 1] - assert "guardrail_choice_index" not in request_data + assert RecordingGuardrail.seen == [None] + + RecordingGuardrail.seen = [] + both_complete: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, + delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]), + finish_reason="tool_calls", + )])] + await handler.process_output_streaming_response( + responses_so_far=both_complete, guardrail_to_apply=guardrail, + litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(), + ) + sent: Final = RecordingGuardrail.seen[-1] + assert sent is not None + assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: From 4cf4510928616722bc852d21d968a4f825a2d1e8 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Thu, 29 Oct 2026 13:31:24 +0100 Subject: [PATCH 14/15] fix(otel): label guardrail spans with the choice they inspected Per-choice guardrail log entries carry choice_index, but the span mapper never read it, so a trace showed several guardrail spans for one request with nothing telling them apart. Expose it as litellm.guardrail.choice_index. --- litellm/integrations/otel/mappers/genai.py | 1 + litellm/integrations/otel/model/payloads.py | 2 ++ litellm/integrations/otel/model/semconv.py | 1 + .../unit/integrations/otel/test_otel_v2_emitter.py | 13 +++++++++++++ 4 files changed, 17 insertions(+) diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 4e07d090edd..132bbf1f401 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -141,6 +141,7 @@ class GenAIMapper: LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score, LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score, LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count, + LiteLLM.GUARDRAIL_CHOICE_INDEX: lambda d: d.choice_index, LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration, LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id, LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template, diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 9c62c45e489..559fa86ef93 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -248,6 +248,7 @@ class GuardrailSpanData: mode: str | None = None status: str | None = None masked_entity_count: int | None = None + choice_index: int | None = None provider: str | None = None action: str | None = None # The guardrail verdict / provider response (e.g. the moderation result), @@ -308,6 +309,7 @@ class GuardrailSpanData: mode=_guardrail_mode_str(get("guardrail_mode")), status=status, masked_entity_count=_total_masked_entities(get("masked_entity_count")), + choice_index=as_int(get("choice_index")), provider=as_str(get("guardrail_provider")), action=as_str(get("guardrail_action")), response_json=_json_or_none(response) if response is not None else None, diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index b7f955a75ed..41a5d85a0bd 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -316,6 +316,7 @@ class LiteLLM: GUARDRAIL_CONFIDENCE_SCORE: Final = "litellm.guardrail.confidence_score" GUARDRAIL_RISK_SCORE: Final = "litellm.guardrail.risk_score" GUARDRAIL_MASKED_ENTITY_COUNT: Final = "litellm.guardrail.masked_entity_count" + GUARDRAIL_CHOICE_INDEX: Final = "litellm.guardrail.choice_index" GUARDRAIL_DURATION: Final = "litellm.guardrail.duration" GUARDRAIL_ID: Final = "litellm.guardrail.id" GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template" diff --git a/tests/unit/integrations/otel/test_otel_v2_emitter.py b/tests/unit/integrations/otel/test_otel_v2_emitter.py index 6e29b2e9c68..403e8f29974 100644 --- a/tests/unit/integrations/otel/test_otel_v2_emitter.py +++ b/tests/unit/integrations/otel/test_otel_v2_emitter.py @@ -344,6 +344,19 @@ def test_guardrail_block_span_is_error_and_carries_verdict(): assert a[LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT] == 2 +def test_guardrail_span_carries_the_choice_index(): + """A per-choice guardrail entry must say on the span which choice it inspected.""" + engine, exporter = _engine() + engine.emit( + SpanRole.GUARDRAIL, + GuardrailSpanData.from_logging_entry( + {"guardrail_name": "g", "guardrail_status": "success", "choice_index": 1} + ), + ) + (span,) = exporter.get_finished_spans() + assert span.attributes[LiteLLM.GUARDRAIL_CHOICE_INDEX] == 1 + + def test_guardrail_success_span_is_unset(): """On success the status is left UNSET (semconv default) — not forced OK.""" engine, exporter = _engine() From 4be43403284c62c0350e0e9d01c1ab40eb57833c Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Thu, 29 Oct 2026 13:59:10 +0100 Subject: [PATCH 15/15] fix(guardrails): keep calls still in flight out of the inspected response Rebuilding an ended stream gives a tool call that never streamed any argument text an empty JSON object, and that response went into request_data, so a guardrail reading it saw a real call taking no arguments. Hand it a copy without those calls, and keep the rebuilt response intact for the rewrites. Parsing alone could not tell the two apart, since an empty object is valid JSON. --- .../chat/guardrail_translation/handler.py | 52 +++++++++++++++---- .../test_openai_guardrail_handler.py | 47 +++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 18c8b5aca9e..9f9485692c2 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -14,6 +14,7 @@ Pattern Overview: This pattern can be replicated for other message formats (e.g., Anthropic). """ +import copy import json import time import uuid @@ -757,6 +758,19 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return False return True + @staticmethod + def _joined_streamed_arguments( + responses_so_far: Sequence["ModelResponseStream"], + ) -> Mapping[tuple[int, int], str]: + """The argument text streamed so far per (choice index, tool call index).""" + fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) + return MappingProxyType( + { + call: "".join(text for other, text in fragments if other == call) + for call in frozenset(call for call, _ in fragments) + } + ) + @staticmethod def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool: """Whether every streamed tool call now carries arguments that parse as JSON. @@ -764,17 +778,33 @@ class OpenAIChatCompletionsHandler(BaseTranslation): A call that has announced itself but emitted no argument text yet is still in flight, so a parallel call finishing first does not make the whole set complete. """ - fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far)) - if not fragments: + joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) + if not joined: return False - joined: Final = MappingProxyType( - { - call: "".join(text for other, text in fragments if other == call) - for call in frozenset(call for call, _ in fragments) - } - ) return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) + @staticmethod + def _without_unfinished_tool_calls( + assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"] + ) -> "ModelResponse": + """Copy of the rebuilt response without the calls still in flight when the stream ended. + + Rebuilding gives a call that never streamed any argument text an empty JSON object, which + reads to a guardrail as a real call taking no arguments rather than an unfinished one. + """ + joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) + pruned: Final = copy.deepcopy(assembled) + for choice in pruned.choices: + streamed = choice.message.tool_calls or () + kept = [ + call + for position, call in enumerate(streamed) + if OpenAIChatCompletionsHandler._parses_as_json(joined.get((choice.index, position), "")) + ] + if len(kept) != len(streamed): + choice.message.tool_calls = kept or None + return pruned + @staticmethod def _rebuild_ended_stream_per_choice( responses_so_far: Sequence["ModelResponseStream"], @@ -903,7 +933,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) ) if assembled is not None: - resolved_request_data["response"] = assembled + resolved_request_data["response"] = ( + assembled + if tool_arguments_complete + else self._without_unfinished_tool_calls(assembled, responses_so_far) + ) if assembled is not None and len(assembled.choices) > 1: choice_rounds: Final = tuple( ( diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 0a0bccceb71..3de123a4241 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1675,6 +1675,53 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert sent is not None assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}'] + @pytest.mark.asyncio + async def test_an_unfinished_parallel_call_is_kept_out_of_what_the_guardrail_reads(self) -> None: + """A call that only announced its name must not reach the guardrail, by either route.""" + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + class RecordingGuardrail(CustomGuardrail): + seen_calls: list[object] = [] + seen_response: list[object] = [] + + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + type(self).seen_calls.append(inputs.get("tool_calls")) + type(self).seen_response.append(request_data.get("response")) + return inputs + + def call(index: int, arguments: str) -> dict[str, object]: + return { + "index": index, "id": f"call_{index}", "type": "function", + "function": {"name": "weather", "arguments": arguments}, + } + + RecordingGuardrail.seen_calls = [] + RecordingGuardrail.seen_response = [] + handler: Final = OpenAIChatCompletionsHandler() + request_data: Final[dict[str, object]] = {} + ended_with_one_unfinished: Final = [ModelResponseStream(choices=[StreamingChoices( + index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]), + finish_reason="tool_calls", + )])] + await handler.process_output_streaming_response( + responses_so_far=ended_with_one_unfinished, + guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"), + litellm_logging_obj=None, request_data=request_data, + stream_transform_sink=StreamTransformSink(), + ) + assert RecordingGuardrail.seen_calls == [None] + inspected: Final = RecordingGuardrail.seen_response[-1] + arguments: Final = tuple( + tool_call.function.arguments + for choice in getattr(inspected, "choices", None) or () + for tool_call in choice.message.tool_calls or () + ) + assert arguments == ('{"city": "Paris"}',) + @pytest.mark.asyncio async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: handler: Final = OpenAIChatCompletionsHandler()