From 54e2a582acd78253f98f77d1f405364b55b98f3a Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:27:18 +0200 Subject: [PATCH 01/12] fix(guardrails): inspect streamed tool calls before delivery --- .../guardrail_translation/base_translation.py | 10 +- .../chat/guardrail_translation/handler.py | 220 +++++++++++------- .../unified_guardrail/unified_guardrail.py | 191 ++++++--------- .../test_openai_guardrail_handler.py | 70 ++++-- .../test_unified_guardrail.py | 180 +++++++++++++- 5 files changed, 433 insertions(+), 238 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 88c4dbfdce6..6280e636700 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional @@ -31,8 +31,12 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: dict[int, str] = field(default_factory=dict) - holdback_per_choice: dict[int, int] = field(default_factory=dict) + mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) + holdback_per_choice: Mapping[int, int] = field(default_factory=dict) + + def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: + self.mutated_text_per_choice = texts + self.holdback_per_choice = holdback @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a12f5f164e4..d18fe36e3df 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -18,9 +18,11 @@ import json import time import uuid from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -46,7 +48,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -168,8 +170,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) + guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ()) guardrailed_tools: Final = guardrailed_inputs.get("tools") if guardrailed_tools is not None: data["tools"] = ( @@ -258,12 +260,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: + for tool in data.get("tools") or (): if isinstance(tool, dict) and tool.get("type") == "function": fn = tool.get("function") if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - for fn in data.get("functions") or []: + for fn in data.get("functions") or (): if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) return names @@ -387,7 +389,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, ) -> ModelResponse: """ Process output response by applying guardrails to text content. @@ -438,9 +440,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # create a standalone dict (SDK / direct-call path). if request_data is None: request_data = {"response": response} - else: - if "response" not in request_data: - request_data["response"] = response + request_data.setdefault("response", response) self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) @@ -460,7 +460,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) @@ -494,7 +494,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, stream_transform_sink: StreamTransformSink | None = None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: @@ -550,14 +550,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path) unless ``deliver_ended_stream_rewrites`` opts the ended-stream branch in.""" - has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) + has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far) if has_stream_ended: await self._process_ended_stream( @@ -623,7 +623,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) # Step 4: Apply guardrailed text back to all streaming chunks # For each choice, replace the combined text across all chunks @@ -656,13 +656,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation): model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) pre_guardrail_texts: Final = self._string_choice_contents(model_response) pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) - await self.process_output_response( - response=model_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=request_data, + inspection_responses: Final = ( + tuple( + model_response.model_copy(update=MappingProxyType({"choices": [choice]})) + for choice in model_response.choices + ) + if pre_guardrail_tool_calls and len(model_response.choices) > 1 + else (model_response,) ) + inspection_request_data: Final = request_data if request_data is not None else {} + try: + for inspection_response in inspection_responses: + inspection_request_data["response"] = inspection_response + inspection_request_data["responses"] = ( + [ + self._narrowed_to_choice(chunk, inspection_response.choices[0].index) + for chunk in responses_so_far + ] + if len(inspection_response.choices) == 1 + else responses_so_far + ) + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + finally: + inspection_request_data["response"] = model_response + inspection_request_data["responses"] = responses_so_far if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -763,7 +786,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, sink: StreamTransformSink, ) -> None: """Run the guardrail over the raw accumulated text and report the @@ -775,8 +798,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far) if not raw_by_index: - sink.mutated_text_per_choice = {} - sink.holdback_per_choice = {} + sink.record(MappingProxyType({}), MappingProxyType({})) return # Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits @@ -788,18 +810,72 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices: Final = sorted(raw_by_index.keys()) texts_to_check: Final = [raw_by_index[i] for i in indices] - if request_data is None: - request_data = {"responses": responses_so_far} - elif "responses" not in request_data: - request_data["responses"] = responses_so_far - self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) + resolved_request_data: Final[dict[str, object]] = ( + request_data if request_data is not None else {"responses": responses_so_far} + ) + if "responses" not in resolved_request_data: + resolved_request_data["responses"] = responses_so_far + self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) + if self._streamed_tool_call_fingerprints(responses_so_far): + assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + resolved_request_data["response"] = assembled + if len(assembled.choices) > 1: + choice_rounds: Final = tuple( + ( + choice, + StreamTransformSink(), + [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], + ) + for choice in assembled.choices + ) + try: + for choice, choice_sink, choice_chunks in choice_rounds: + resolved_request_data["response"] = assembled.model_copy( + update=MappingProxyType({"choices": [choice]}) + ) + resolved_request_data["responses"] = choice_chunks + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + finally: + resolved_request_data["response"] = assembled + resolved_request_data["responses"] = responses_so_far + merged_texts: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + merged_holdback: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + sink.record(merged_texts, merged_holdback) + return + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) + ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, - request_data=request_data, + request_data=resolved_request_data, input_type="response", logging_obj=litellm_logging_obj, ) @@ -819,13 +895,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): len(texts_to_check), ) - holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or [] - sink.mutated_text_per_choice = { - idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts) - } - sink.holdback_per_choice = { - indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) - } + holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () + transformed_texts: Final = MappingProxyType( + {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} + ) + transformed_holdback: Final = MappingProxyType( + {indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)} + ) + sink.record(transformed_texts, transformed_holdback) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) @@ -1145,20 +1222,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _function_tool_call_fragments( responses_so_far: Sequence["ModelResponseStream"], ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: - """Group the stream's function tool-call fragments by their tool-call index, in - the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping - only the indices the builder keeps (an id and a name somewhere in the stream).""" fragments: Final = tuple( - tool_call + (choice.index, tool_call) for response in responses_so_far for choice in response.choices for tool_call in choice.delta.tool_calls or () if isinstance(tool_call, ChatCompletionDeltaToolCall) ) - identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) - named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id) + named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name) return tuple( - tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key) + for key in sorted(identified & named) ) def _write_ended_stream_tool_call_rewrites( @@ -1168,28 +1243,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], guardrail_name: str, ) -> None: - """Write ended-stream guardrail tool-call rewrites back across the buffered - chunks: the rewritten name and full arguments land in the tool call's first - fragment and the arguments of its later fragments are blanked, mirroring the - text write-back. A rewrite on a stream carrying more than one distinct choice - index, or whose fragments do not line up with the rebuilt tool calls, is - reported as undeliverable, so the pipeline executor discards it and releases - the original chunks.""" post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - stream_choice_indices: Final = frozenset( - choice.index for response in responses_so_far for choice in response.choices - ) fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) - if len(stream_choice_indices) != 1: - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite( - guardrail_name, - f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written " - "back on single-choice streams", - ) if len(fragments_by_tool_call) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite @@ -1212,7 +1269,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], - guardrailed_texts: list[str], + guardrailed_texts: Sequence[str], task_mappings: list[tuple[int, int | None]], ) -> None: """ @@ -1230,12 +1287,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Override this method to customize how responses are applied to streaming responses. """ # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx) - guardrail_map: Final[dict[tuple[int, int | None], str]] = {} - for task_idx, guardrail_response in enumerate(guardrailed_texts): - mapping = task_mappings[task_idx] - choice_idx = cast(int, mapping[0]) - content_idx_optional = cast(int | None, mapping[1]) - guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response + guardrail_map: Final = MappingProxyType( + {task_mappings[index]: text for index, text in enumerate(guardrailed_texts)} + ) # Track which choices we've already set the guardrailed text for # Key: (choice_idx, content_idx), Value: boolean (True if already set) @@ -1245,34 +1299,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # choice by its index field: on n>1 streams a chunk usually carries one # choice at list position 0 whose index names the logical choice. for response in responses: - for choice in response.choices: - if isinstance(choice, litellm.StreamingChoices): - content = choice.delta.content - elif isinstance(choice, litellm.Choices): - content = choice.message.content - else: - continue - + for choice, message in ( + (choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message) + for choice in response.choices + if isinstance(choice, (StreamingChoices, Choices)) + ): + content = message.content if content is None: continue if isinstance(content, str): - # String content str_key: tuple[int, int | None] = (choice.index, None) if str_key in guardrail_map: - if str_key not in already_set: - # First chunk - set the complete guardrailed text - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = guardrail_map[str_key] - elif isinstance(choice, litellm.Choices): - choice.message.content = guardrail_map[str_key] - already_set[str_key] = True - else: - # Subsequent chunks - clear the content - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = "" - elif isinstance(choice, litellm.Choices): - choice.message.content = "" + message.content = guardrail_map[str_key] if str_key not in already_set else "" + already_set[str_key] = True elif isinstance(content, list): # List content - handle each content item diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index a7f813d953e..9134a1cd0f3 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias, cast, runtime import anyio from fastapi import HTTPException +from pydantic import InstanceOf, TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.caching.caching import DualCache @@ -49,6 +50,7 @@ if TYPE_CHECKING: A2A_CALL_TYPES: Final = (CallTypes.asend_message, CallTypes.send_message) GUARDRAIL_NAME: Final = "unified_llm_guardrails" +GUARDRAIL_ADAPTER: Final[TypeAdapter[CustomGuardrail | None]] = TypeAdapter(InstanceOf[CustomGuardrail] | None) _RequestData: TypeAlias = dict[str, object] @@ -513,9 +515,9 @@ class UnifiedLLMGuardrails(CustomLogger): self, *, reference_chunk: object, - mutated_text_per_choice: dict[int, str], + mutated_text_per_choice: Mapping[int, str], emitted_text_per_choice: dict[int, str], - holdback_per_choice: dict[int, int], + holdback_per_choice: Mapping[int, int], finish_reason_per_choice: dict[int, str | None], held_chars_per_choice: dict[int, int], is_final: bool, @@ -751,46 +753,13 @@ class UnifiedLLMGuardrails(CustomLogger): saw_tool_calls = False saw_text_content = False - tool_calls_released = False # rebind-ok: set once a raw tool call reaches the client unscanned - end_of_stream_inspection_started = False # rebind-ok: set once the end-of-stream inspection owns the verdict try: async for item in response: - # v1 transforms only text. A chunk carrying tool_calls is passed - # through raw so function-calling turns are not dropped, but ONLY - # its tool-call fields are forwarded: content is stripped so any - # response text (in the same delta, or in another choice of an n>1 - # chunk) can never bypass the transform. The original chunk is kept - # in responses_so_far so its text is still accumulated + redacted + - # emitted as synthetic deltas, and so the guardrail inspects the - # assembled tool calls at end of stream (see the block inspection - # below), matching block_only. finish_reason rides on the raw - # tool-only chunk, so it is not recorded for the text flush. if self._chunk_has_tool_calls(item): saw_tool_calls = True responses_so_far.append(item) last_chunk = item - # Fix #3 — flush accumulated text BEFORE the tool-call - # passthrough. Without this, a stream of text chunks that - # hasn't yet hit a sampled round can be trailed by a - # tool-call chunk carrying finish_reason="tool_calls"; an - # SSE-compliant client stops reading at that finish_reason - # and drops the end-of-stream text flush that would follow. - if saw_text_content: - async for out in _round(item, is_final=False): - yield out - # Fix #1 — pass finish_reason_per_choice into the - # passthrough so a mixed content+tool_call chunk defers its - # finish_reason to the final text terminator (see the - # _tool_call_passthrough_chunk docstring). - tool_only = self._tool_call_passthrough_chunk( - item, - finish_reason_per_choice=finish_reason_per_choice, - held_choices=_held_choices(held_chars_per_choice), - ) - responses_yielded.append(tool_only) - tool_calls_released = True - yield tool_only continue if self._is_trailing_metadata_chunk(item): @@ -808,39 +777,62 @@ class UnifiedLLMGuardrails(CustomLogger): # sampled round here would guardrail the same content twice. if ( not end_of_stream_only + and not saw_tool_calls and not self._chunk_has_finish_reason(item) and chunk_counter % sampling_rate == 0 ): async for out in _round(item, is_final=False): yield out - # v1 does not transform streamed tool calls, but they must still go - # through the guardrail's block decision. Run the block_only inspection - # over the full assembled response so tool calls cannot bypass it. - # - # Pass a deep copy of responses_so_far — the block path routes through - # ``_process_streaming_block_only`` which mutates ``delta.content`` - # in-place on the chunk objects it receives. For an n>1 chunk carrying - # text on one choice and tool_calls (with finish_reason) on another, - # ``has_stream_ended`` reads ``choices[0]`` alone and can miss the - # terminal signal, letting the block path rewrite the raw accumulator. - # The subsequent final ``_round`` would then re-read the already-mutated - # text, producing double-application for a non-idempotent guardrail or a - # ``stream_transform_underflow`` 400 from mismatched prefixes. A shallow - # list copy wouldn't help — the mutation is on the chunk objects - # themselves — so we deepcopy. - end_of_stream_inspection_started = True if saw_tool_calls: - async for out in self._inspect_full_response_for_block( + inspected_responses: Final = copy.deepcopy(responses_so_far) + async for out in self._inspect_full_response( endpoint_translation=endpoint_translation, guardrail_to_apply=guardrail_to_apply, request_data=request_data, user_api_key_dict=user_api_key_dict, - responses_so_far=copy.deepcopy(responses_so_far), + responses_so_far=inspected_responses, responses_yielded=responses_yielded, ): yield out + if saw_text_content: + async for out in _round(last_chunk, is_final=False): + yield out + tool_chunks: Final = tuple( + self._tool_call_passthrough_chunk( + buffered_item, + finish_reason_per_choice=finish_reason_per_choice, + held_choices=_held_choices(held_chars_per_choice), + ) + for buffered_item in inspected_responses + if self._chunk_has_tool_calls(buffered_item) + ) + + async def checked_tail() -> AsyncGenerator[object, None]: + try: + async for tail_chunk in self._emit_stream_tail( + last_chunk=last_chunk, + final_round=_round, + responses_so_far=responses_so_far, + responses_yielded=responses_yielded, + ): + yield tail_chunk + except _StreamTerminated as exc: + yield exc + + tail_chunks: Final = tuple([chunk async for chunk in checked_tail()]) + if tail_chunks and isinstance(tail_chunks[-1], _StreamTerminated): + for error_chunk in tail_chunks[:-1]: + yield error_chunk + return + for tool_only in tool_chunks: + responses_yielded.append(tool_only) + yield tool_only + for tail_chunk in tail_chunks: + yield tail_chunk + return + async for out in self._emit_stream_tail( last_chunk=last_chunk, final_round=_round, @@ -850,37 +842,6 @@ class UnifiedLLMGuardrails(CustomLogger): yield out except _StreamTerminated: return - except (GeneratorExit, asyncio.CancelledError): - await self._scan_uninspected_tool_calls_after_disconnect( - uninspected=tool_calls_released and not end_of_stream_inspection_started and not terminated.is_set(), - endpoint_translation=endpoint_translation, - responses_released=responses_yielded, - guardrail_to_apply=guardrail_to_apply, - user_api_key_dict=user_api_key_dict, - request_data=request_data, - ) - raise - - @staticmethod - async def _scan_uninspected_tool_calls_after_disconnect( - *, - uninspected: bool, - endpoint_translation: _EndpointTranslation, - responses_released: Sequence[object], - guardrail_to_apply: CustomGuardrail, - user_api_key_dict: UserAPIKeyAuth, - request_data: _RequestData, - ) -> None: - if not uninspected: - return - await UnifiedLLMGuardrails._scan_released_stream_after_disconnect( - endpoint_translation=endpoint_translation, - responses_released=responses_released, - last_scan_key=None, - guardrail_to_apply=guardrail_to_apply, - user_api_key_dict=user_api_key_dict, - request_data=request_data, - ) async def _emit_stream_tail( self, @@ -899,29 +860,21 @@ class UnifiedLLMGuardrails(CustomLogger): responses_yielded.append(trailing) yield trailing - async def _inspect_full_response_for_block( + async def _inspect_full_response( self, *, endpoint_translation: _EndpointTranslation, guardrail_to_apply: CustomGuardrail, - request_data: dict, + request_data: _RequestData, user_api_key_dict: UserAPIKeyAuth, responses_so_far: Sequence[object], responses_yielded: Sequence[object], ) -> AsyncGenerator[object, None]: - """Run the block-only guardrail inspection over the full assembled - response (text + tool calls) so nothing bypasses the block decision. - - The guardrail's returned transforms are discarded here (v1 does not - transform tool calls); only its block decision matters. A block is - surfaced the same way as elsewhere: ModifyResponseException terminates the - stream via the shared block handler; a GenericGuardrailAPI block raises and - propagates, matching block_only. - """ from litellm.integrations.custom_guardrail import ModifyResponseException + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite try: - with anyio.CancelScope(shield=bool(responses_yielded)): + with anyio.CancelScope(shield=True): await endpoint_translation.process_output_streaming_response( responses_so_far=responses_so_far, guardrail_to_apply=guardrail_to_apply, @@ -929,7 +882,10 @@ class UnifiedLLMGuardrails(CustomLogger): user_api_key_dict=user_api_key_dict, request_data=request_data, stream_transform_sink=None, + deliver_ended_stream_rewrites=True, ) + except UndeliverableStreamRewrite as exc: + raise HTTPException(status_code=400, detail="Guardrail stream rewrite could not be applied") from exc except ModifyResponseException as e: if e.original_response is None: e.original_response = responses_so_far @@ -1113,12 +1069,15 @@ class UnifiedLLMGuardrails(CustomLogger): # litellm.integrations.custom_guardrail. from litellm.integrations.custom_guardrail import ModifyResponseException - if guardrail_to_apply is None: - guardrail_to_apply = request_data.pop("guardrail_to_apply", None) + resolved_guardrail: Final[CustomGuardrail | None] = ( + guardrail_to_apply + if guardrail_to_apply is not None + else GUARDRAIL_ADAPTER.validate_python(request_data.pop("guardrail_to_apply", None)) + ) typed_request_data: Final[_RequestData] = request_data def _streaming_flag(name: str, default: object) -> Any: - return self.resolve_streaming_flag(guardrail_to_apply, name, default) + return self.resolve_streaming_flag(resolved_guardrail, name, default) sampling_rate: Final[int] = _streaming_flag("streaming_sampling_rate", 5) # Only apply the guardrail at end of stream (not per chunk). @@ -1141,30 +1100,30 @@ class UnifiedLLMGuardrails(CustomLogger): if ( buffer_until_moderated - and guardrail_to_apply is not None - and getattr(guardrail_to_apply, "mask_response_content", False) + and resolved_guardrail is not None + and getattr(resolved_guardrail, "mask_response_content", False) ): verbose_proxy_logger.warning( "UnifiedLLMGuardrails: streaming_buffer_until_moderated is disabled for %s " "because mask_response_content=True -- buffered replay would release " "unredacted original chunks instead of the moderated output.", - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) buffer_until_moderated = False if buffer_until_moderated and not release_on_scan: end_of_stream_only = True - if guardrail_to_apply is None: + if resolved_guardrail is None: async for item in response: yield item return event_type: Final[GuardrailEventHooks] = GuardrailEventHooks.post_call - if guardrail_to_apply.should_run_guardrail(data=request_data, event_type=event_type) is not True: + if resolved_guardrail.should_run_guardrail(data=request_data, event_type=event_type) is not True: verbose_proxy_logger.debug( "UnifiedLLMGuardrails: Post-call streaming scanning disabled for %s", - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) async for item in response: yield item @@ -1184,7 +1143,7 @@ class UnifiedLLMGuardrails(CustomLogger): if transform_call_type is not None: async with contextlib.aclosing( self._run_incremental_transform_stream( - guardrail_to_apply=guardrail_to_apply, + guardrail_to_apply=resolved_guardrail, response=response, request_data=typed_request_data, user_api_key_dict=user_api_key_dict, @@ -1201,7 +1160,7 @@ class UnifiedLLMGuardrails(CustomLogger): "UnifiedLLMGuardrails: streaming_transform_mode=incremental_diff is only supported " "for the OpenAI chat completions streaming path with a resolvable request route; " "falling back to block_only for %s", - getattr(guardrail_to_apply, "guardrail_name", None), + getattr(resolved_guardrail, "guardrail_name", None), ) # Infer call type from first chunk @@ -1273,7 +1232,7 @@ class UnifiedLLMGuardrails(CustomLogger): verbose_proxy_logger.debug( "Skipping streaming chunk %s for guardrail %s: nothing new to scan since the last round", chunk_counter, - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) if buffer_until_moderated: if hold_window: @@ -1293,7 +1252,7 @@ class UnifiedLLMGuardrails(CustomLogger): "Processing streaming chunk %s (sampling_rate=%s) with guardrail %s", chunk_counter, sampling_rate, - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) original_items = ( @@ -1303,7 +1262,7 @@ class UnifiedLLMGuardrails(CustomLogger): try: await endpoint_translation.process_output_streaming_response( responses_so_far=responses_so_far, - guardrail_to_apply=guardrail_to_apply, + guardrail_to_apply=resolved_guardrail, litellm_logging_obj=request_data.get("litellm_logging_obj"), user_api_key_dict=user_api_key_dict, request_data=request_data, @@ -1349,7 +1308,7 @@ class UnifiedLLMGuardrails(CustomLogger): verbose_proxy_logger.debug( "Holding %s buffered chunks for guardrail %s: this round could not scan the whole window", len(withheld_items), - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) withheld_items[:] = original_items continue @@ -1369,7 +1328,7 @@ class UnifiedLLMGuardrails(CustomLogger): verbose_proxy_logger.debug( "Processing final streaming response with all %s chunks for guardrail %s", len(responses_so_far), - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) endpoint_translation = mappings[CallTypes(call_type)]() @@ -1386,7 +1345,7 @@ class UnifiedLLMGuardrails(CustomLogger): if _is_redundant_scan(end_scan_key, last_scan_key): verbose_proxy_logger.debug( "Skipping end-of-stream scan for guardrail %s: the last sampled round already scanned it all", - guardrail_to_apply.guardrail_name, + resolved_guardrail.guardrail_name, ) for buffered_item in buffered_items or (): yield buffered_item @@ -1399,7 +1358,7 @@ class UnifiedLLMGuardrails(CustomLogger): with anyio.CancelScope(shield=chunks_yielded): await endpoint_translation.process_output_streaming_response( responses_so_far=responses_so_far, - guardrail_to_apply=guardrail_to_apply, + guardrail_to_apply=resolved_guardrail, litellm_logging_obj=request_data.get("litellm_logging_obj"), user_api_key_dict=user_api_key_dict, request_data=request_data, @@ -1443,13 +1402,13 @@ class UnifiedLLMGuardrails(CustomLogger): chunks_yielded and not verdict_settled and translation_class is not None - and isinstance(guardrail_to_apply, CustomGuardrail) + and isinstance(resolved_guardrail, CustomGuardrail) ): await self._scan_released_stream_after_disconnect( endpoint_translation=translation_class(), responses_released=responses_yielded, last_scan_key=last_scan_key, - guardrail_to_apply=guardrail_to_apply, + guardrail_to_apply=resolved_guardrail, user_api_key_dict=user_api_key_dict, request_data=typed_request_data, ) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..bffe6e4c123 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Literal, Optional +from typing import Any, Final, Literal, Optional import pytest @@ -1395,29 +1395,65 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert [ - (tool_call["id"], tool_call["function"]["arguments"]) - for tool_call in guardrail.seen_inputs[-1]["tool_calls"] - ] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')] + [(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]] + for inputs in guardrail.seen_inputs + ] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]] @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + @pytest.mark.parametrize("transform", [False, True]) + async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None: + from fastapi import HTTPException + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - handler = OpenAIChatCompletionsHandler() - chunks = self._two_choice_tool_call_stream_chunks() + class ContextGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index) + if response.choices[0].message.content == "forbidden": + raise HTTPException(status_code=400, detail="later choice blocked") + return inputs - with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised: - await handler.process_output_streaming_response( - responses_so_far=chunks, - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, - deliver_ended_stream_rewrites=True, + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("allowed", "forbidden"))])] + request_data: Final = {"metadata": {"trace": "retained"}} + with pytest.raises(HTTPException, match="later choice blocked"): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink() if transform else None, ) + assert request_data["visited_choices"] == (0, 1) + assert request_data["metadata"]["trace"] == "retained" + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) - assert raised.value.guardrail_name == "test" - assert raised.value.reason == ( - "the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams" + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: + handler: Final = OpenAIChatCompletionsHandler() + chunks: Final = self._two_choice_tool_call_stream_chunks() + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, ) + arguments: Final = tuple( + "".join(call.function.arguments for chunk in chunks for choice in chunk.choices + if choice.index == index for call in choice.delta.tool_calls or ()) + for index in range(2) + ) + assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}') @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..4dd2f2cb6e3 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -8,6 +8,8 @@ from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal +from pydantic import TypeAdapter + import anyio import pytest @@ -50,6 +52,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import ( CallTypes, CallTypesLiteral, + ChatCompletionMessageToolCall, Delta, GenericGuardrailAPIInputs, ModelResponse, @@ -1008,6 +1011,42 @@ def _delta_text(item): return item.choices[0].delta.content or "" +class _ToolRedactingGuardrail(CustomGuardrail): + def __init__(self, require_tool_context: bool = False) -> None: + super().__init__(guardrail_name="tool-redactor", event_hook=GuardrailEventHooks.post_call, default_on=True) + self.streaming_transform_mode = "incremental_diff" + self.streaming_sampling_rate = 1 + self.require_tool_context = require_tool_context + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: object | None = None, + ) -> GenericGuardrailAPIInputs: + calls: Final = TypeAdapter(tuple[ChatCompletionMessageToolCall, ...]).validate_python( + inputs.get("tool_calls", ()) + ) + input_texts: Final = inputs.get("texts", ()) + texts: Final = tuple( + "checked:" + text.replace("SECRET", "MASKED") + if not self.require_tool_context or (calls and index == len(input_texts) - 1) else text + for index, text in enumerate(input_texts) + ) + return { + **inputs, + "texts": list(texts), + "stream_holdback_chars": [len(text) for text in texts], + "tool_calls": [ + call.model_copy(update={"function": call.function.model_copy(update={ + "arguments": call.function.arguments.replace("SECRET", "MASKED"), + })}) + for call in calls + ], + } + + class TestStreamingTransform: """Streaming text-transformation (incremental_diff) path on the OpenAI chat completions streaming surface.""" @@ -1016,6 +1055,103 @@ class TestStreamingTransform: def _use_openai_handler_mapping(self, monkeypatch): _patch_translation_mappings(monkeypatch, {CallTypes.acompletion: OpenAIChatCompletionsHandler}) + @pytest.mark.asyncio + @pytest.mark.parametrize("require_tool_context", [False, True]) + @pytest.mark.parametrize("include_text", [False, True]) + @pytest.mark.parametrize("tool_count", [1, 2]) + async def test_buffered_tool_arguments_are_rewritten_before_delivery( + self, include_text: bool, tool_count: int, require_tool_context: bool + ) -> None: + chunks: Final = ( + *([_stream_chunk("hello SECRET")] if include_text else []), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(tool_calls=[ + {"index": index, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": '{"contact":"SEC'}} + for index in range(tool_count) + ]))]), + ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(tool_calls=[ + {"index": index, "function": {"arguments": 'RET"}'}} for index in range(tool_count) + ]), finish_reason="tool_calls")]), + ModelResponseStream(choices=[], usage={"prompt_tokens": 7, "completion_tokens": 11, "total_tokens": 18}), + ) + out: Final = await _drive_stream( + UnifiedLLMGuardrails(), _ToolRedactingGuardrail(require_tool_context=require_tool_context), chunks + ) + calls: Final = tuple( + call for chunk in out for choice in chunk.choices for call in choice.delta.tool_calls or () + ) + for index in range(tool_count): + arguments: Final = "".join(call.function.arguments or "" for call in calls if call.index == index) + assert arguments == '{"contact":"MASKED"}' + assert next(call.id for call in calls if call.index == index and call.id) == f"call_{index}" + assert "".join(_delta_text(chunk) for chunk in out) == ("checked:hello MASKED" if include_text else "") + assert any(choice.finish_reason == "tool_calls" for chunk in out for choice in chunk.choices) + assert out[-1].usage.total_tokens == 18 + assert all("SECRET" not in chunk.model_dump_json() for chunk in out) + + @pytest.mark.asyncio + @pytest.mark.parametrize("tool_choice_index", [0, 1]) + async def test_tool_dependent_text_rewrites_keep_choice_context(self, tool_choice_index: int) -> None: + chunks: Final = ( + ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content="SECRET" if index == tool_choice_index else "plain"), + ) for index in (1, 0)]), + ModelResponseStream(choices=[StreamingChoices( + index=tool_choice_index, + delta=Delta(tool_calls=[{ + "index": 0, "id": "call_context", "type": "function", + "function": {"name": "contact", "arguments": '{"contact":"SECRET"}'}, + }]), finish_reason="tool_calls", + )]), + ) + out: Final = await _drive_stream(UnifiedLLMGuardrails(), _ToolRedactingGuardrail(True), chunks) + for index in (0, 1): + text: Final = "".join( + choice.delta.content or "" for chunk in out for choice in chunk.choices if choice.index == index + ) + assert text == ("checked:MASKED" if index == tool_choice_index else "plain") + assert all("SECRET" not in chunk.model_dump_json() for chunk in out) + + @pytest.mark.asyncio + async def test_tool_rewrites_keep_completion_choices_separate(self) -> None: + async def response() -> AsyncIterator[ModelResponseStream]: + yield ModelResponseStream(choices=[StreamingChoices( + index=index, + delta=Delta(tool_calls=[{"index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": '{"contact":"SECRET"}'}}]), + finish_reason="tool_calls", + ) for index in range(2)]) + + iterator: Final = UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/chat/completions"), + response=response(), request_data={"guardrail_to_apply": _ToolRedactingGuardrail()}, + ) + chunks: Final = tuple([chunk async for chunk in iterator]) + calls: Final = tuple( + (choice.index, call.id, call.function.arguments) + for chunk in chunks for choice in chunk.choices for call in choice.delta.tool_calls or () + ) + assert calls == ((0, "call_0", '{"contact":"MASKED"}'), (1, "call_1", '{"contact":"MASKED"}')) + + @pytest.mark.asyncio + async def test_undeliverable_rewrite_is_a_closed_failure(self) -> None: + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + class UndeliverableTranslation(OpenAIChatCompletionsHandler): + async def process_output_streaming_response( + self, *args: object, **kwargs: object + ) -> list[ModelResponseStream]: + raise UndeliverableStreamRewrite("tool-redactor", "unmappable tool fragments") + + iterator: Final = UnifiedLLMGuardrails()._inspect_full_response( + endpoint_translation=UndeliverableTranslation(), guardrail_to_apply=_ToolRedactingGuardrail(), + request_data={}, user_api_key_dict=UserAPIKeyAuth(), responses_so_far=(), responses_yielded=(), + ) + with pytest.raises(unified_module.HTTPException) as error: + await anext(iterator) + assert error.value.status_code == 400 + assert error.value.detail == "Guardrail stream rewrite could not be applied" + @pytest.mark.asyncio async def test_block_only_drops_text_rewrites(self): """Default block_only: the guardrail's uppercasing never reaches the @@ -1436,14 +1572,15 @@ class TestStreamingTransform: assert out[-2].choices[0].finish_reason == "stop" @pytest.mark.asyncio - async def test_tool_call_blocking_guardrail_is_enforced(self): + @pytest.mark.parametrize(("content", "allowed_scans"), [(None, 0), ("proposal", 0), ("proposal", 1)]) + async def test_tool_call_blocking_guardrail_is_enforced(self, content: str | None, allowed_scans: int): """A guardrail that blocks on tool calls must terminate the incremental_diff stream: tool calls go through the block decision, not bypass it.""" from litellm.exceptions import GuardrailRaisedException class _ToolCallBlocker(_StreamingTextGuardrail): async def apply_guardrail(self, inputs, request_data, input_type, **kwargs): - if input_type == "response" and inputs.get("tool_calls"): + if input_type == "response" and self.response_calls >= allowed_scans: raise GuardrailRaisedException( guardrail_name="tc-block", message="blocked tool call", @@ -1456,7 +1593,7 @@ class TestStreamingTransform: StreamingChoices( index=0, delta=Delta( - content=None, + content=content, tool_calls=[ { "index": 0, @@ -1471,8 +1608,21 @@ class TestStreamingTransform: ], ) + async def upstream(): + yield tool_chunk + + stream: Final = UnifiedLLMGuardrails().async_post_call_streaming_iterator_hook( + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", request_route="/v1/chat/completions"), + response=upstream(), + request_data={"guardrail_to_apply": _ToolCallBlocker(), "model": "gpt-4"}, + ) + async def consume_checked_stream() -> None: + async for chunk in stream: + assert isinstance(chunk, ModelResponseStream) + assert all(not choice.delta.tool_calls and choice.finish_reason is None for choice in chunk.choices) + with pytest.raises(GuardrailRaisedException): - await _drive_stream(UnifiedLLMGuardrails(), _ToolCallBlocker(), [tool_chunk]) + await consume_checked_stream() @pytest.mark.asyncio async def test_mixed_content_and_tool_call_chunk_does_not_leak_text(self): @@ -2690,7 +2840,7 @@ class TestStreamingClientDisconnectScan: yield _stream_chunk(" tail", finish_reason="stop") @pytest.mark.asyncio - async def test_closing_while_an_incremental_diff_block_is_delivered_does_not_inspect_the_tool_call_again(self): + async def test_incremental_diff_block_withholds_tool_calls_and_does_not_inspect_again(self): guardrail = _MarkerBlockingStreamingTextGuardrail() async def upstream() -> AsyncIterator[ModelResponseStream]: @@ -2704,12 +2854,16 @@ class TestStreamingClientDisconnectScan: released = [await stream.__anext__(), await stream.__anext__()] await stream.aclose() - assert [call.function.name for call in released[0].choices[0].delta.tool_calls] == ["get_weather"] - assert guardrail.received_texts == [["BLOCKME"]], guardrail.received_texts - assert guardrail.received_tool_calls == [], guardrail.received_tool_calls + assert all(isinstance(item, bytes) for item in released) + assert b"get_weather" not in b"".join(released) + assert b"Paris" not in b"".join(released) + assert guardrail.received_texts == [["BLOCKME tail"]], guardrail.received_texts + assert [[call["function"]["name"] for call in calls] for calls in guardrail.received_tool_calls] == [ + ["get_weather"] + ] @pytest.mark.asyncio - async def test_closing_after_a_released_tool_call_under_incremental_diff_scans_no_held_back_text(self): + async def test_closing_after_a_buffered_tool_call_inspects_held_text_before_delivery(self): guardrail = _StreamingTextGuardrail(holdback_schedule=[len("held secret")] * 2) async def upstream() -> AsyncIterator[ModelResponseStream]: @@ -2718,12 +2872,14 @@ class TestStreamingClientDisconnectScan: yield tool_chunk stream = self._guarded_stream(guardrail, upstream()) - received = await stream.__anext__() + received: Final = tuple([await stream.__anext__() for _ in range(2)]) + scans_before_close: Final = len(guardrail.received_texts) await stream.aclose() - assert [call.function.name for call in received.choices[0].delta.tool_calls] == ["get_weather"] + assert [call.function.name for call in received[-1].choices[0].delta.tool_calls] == ["get_weather"] assert guardrail.received_tool_calls, guardrail.received_texts - assert all("held secret" not in text for text in guardrail.received_texts[-1]), guardrail.received_texts + assert guardrail.received_texts[0] == ["held secret"] + assert len(guardrail.received_texts) == scans_before_close def _responses_delta(sequence_number, text): From 95348ba610fcd21b742c64786d1fa0eed7d4b9f4 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:36:57 +0200 Subject: [PATCH 02/12] fix(guardrails): isolate streamed completion choices during inspection --- .../guardrail_translation/base_translation.py | 10 +- .../chat/guardrail_translation/handler.py | 220 +++++++++++------- .../test_openai_guardrail_handler.py | 70 ++++-- 3 files changed, 190 insertions(+), 110 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 88c4dbfdce6..6280e636700 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional @@ -31,8 +31,12 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: dict[int, str] = field(default_factory=dict) - holdback_per_choice: dict[int, int] = field(default_factory=dict) + mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) + holdback_per_choice: Mapping[int, int] = field(default_factory=dict) + + def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: + self.mutated_text_per_choice = texts + self.holdback_per_choice = holdback @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index a12f5f164e4..d18fe36e3df 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -18,9 +18,11 @@ import json import time import uuid from collections.abc import Iterator, Mapping, Sequence +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast +from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -46,7 +48,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -168,8 +170,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) - guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) + guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ()) guardrailed_tools: Final = guardrailed_inputs.get("tools") if guardrailed_tools is not None: data["tools"] = ( @@ -258,12 +260,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] - for tool in data.get("tools") or []: + for tool in data.get("tools") or (): if isinstance(tool, dict) and tool.get("type") == "function": fn = tool.get("function") if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - for fn in data.get("functions") or []: + for fn in data.get("functions") or (): if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) return names @@ -387,7 +389,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, ) -> ModelResponse: """ Process output response by applying guardrails to text content. @@ -438,9 +440,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # create a standalone dict (SDK / direct-call path). if request_data is None: request_data = {"response": response} - else: - if "response" not in request_data: - request_data["response"] = response + request_data.setdefault("response", response) self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) @@ -460,7 +460,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) @@ -494,7 +494,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict | None = None, + request_data: dict[str, object] | None = None, stream_transform_sink: StreamTransformSink | None = None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: @@ -550,14 +550,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path) unless ``deliver_ended_stream_rewrites`` opts the ended-stream branch in.""" - has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) + has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far) if has_stream_ended: await self._process_ended_stream( @@ -623,7 +623,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) # Step 4: Apply guardrailed text back to all streaming chunks # For each choice, replace the combined text across all chunks @@ -656,13 +656,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation): model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) pre_guardrail_texts: Final = self._string_choice_contents(model_response) pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) - await self.process_output_response( - response=model_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=request_data, + inspection_responses: Final = ( + tuple( + model_response.model_copy(update=MappingProxyType({"choices": [choice]})) + for choice in model_response.choices + ) + if pre_guardrail_tool_calls and len(model_response.choices) > 1 + else (model_response,) ) + inspection_request_data: Final = request_data if request_data is not None else {} + try: + for inspection_response in inspection_responses: + inspection_request_data["response"] = inspection_response + inspection_request_data["responses"] = ( + [ + self._narrowed_to_choice(chunk, inspection_response.choices[0].index) + for chunk in responses_so_far + ] + if len(inspection_response.choices) == 1 + else responses_so_far + ) + await self.process_output_response( + response=inspection_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=inspection_request_data, + ) + finally: + inspection_request_data["response"] = model_response + inspection_request_data["responses"] = responses_so_far if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -763,7 +786,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict | None, + request_data: dict[str, object] | None, sink: StreamTransformSink, ) -> None: """Run the guardrail over the raw accumulated text and report the @@ -775,8 +798,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far) if not raw_by_index: - sink.mutated_text_per_choice = {} - sink.holdback_per_choice = {} + sink.record(MappingProxyType({}), MappingProxyType({})) return # Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits @@ -788,18 +810,72 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices: Final = sorted(raw_by_index.keys()) texts_to_check: Final = [raw_by_index[i] for i in indices] - if request_data is None: - request_data = {"responses": responses_so_far} - elif "responses" not in request_data: - request_data["responses"] = responses_so_far - self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) + resolved_request_data: Final[dict[str, object]] = ( + request_data if request_data is not None else {"responses": responses_so_far} + ) + if "responses" not in resolved_request_data: + resolved_request_data["responses"] = responses_so_far + self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) + if self._streamed_tool_call_fingerprints(responses_so_far): + assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) + resolved_request_data["response"] = assembled + if len(assembled.choices) > 1: + choice_rounds: Final = tuple( + ( + choice, + StreamTransformSink(), + [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], + ) + for choice in assembled.choices + ) + try: + for choice, choice_sink, choice_chunks in choice_rounds: + resolved_request_data["response"] = assembled.model_copy( + update=MappingProxyType({"choices": [choice]}) + ) + resolved_request_data["responses"] = choice_chunks + await self._process_streaming_transform( + responses_so_far=choice_chunks, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=resolved_request_data, + sink=choice_sink, + ) + finally: + resolved_request_data["response"] = assembled + resolved_request_data["responses"] = responses_so_far + merged_texts: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + merged_holdback: Final = MappingProxyType( + dict( + chain.from_iterable( + choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds + ) + ) + ) + sink.record(merged_texts, merged_holdback) + return + tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) + inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( + tuple( + MappingProxyType({"index": index, **converted}) + for index, tool_call in enumerate(tool_calls) + if (converted := self._convert_tool_call_to_dict(tool_call)) is not None + ) + ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, - request_data=request_data, + request_data=resolved_request_data, input_type="response", logging_obj=litellm_logging_obj, ) @@ -819,13 +895,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): len(texts_to_check), ) - holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or [] - sink.mutated_text_per_choice = { - idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts) - } - sink.holdback_per_choice = { - indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) - } + holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () + transformed_texts: Final = MappingProxyType( + {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} + ) + transformed_holdback: Final = MappingProxyType( + {indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)} + ) + sink.record(transformed_texts, transformed_holdback) def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) @@ -1145,20 +1222,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _function_tool_call_fragments( responses_so_far: Sequence["ModelResponseStream"], ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: - """Group the stream's function tool-call fragments by their tool-call index, in - the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping - only the indices the builder keeps (an id and a name somewhere in the stream).""" fragments: Final = tuple( - tool_call + (choice.index, tool_call) for response in responses_so_far for choice in response.choices for tool_call in choice.delta.tool_calls or () if isinstance(tool_call, ChatCompletionDeltaToolCall) ) - identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) - named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id) + named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name) return tuple( - tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key) + for key in sorted(identified & named) ) def _write_ended_stream_tool_call_rewrites( @@ -1168,28 +1243,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], guardrail_name: str, ) -> None: - """Write ended-stream guardrail tool-call rewrites back across the buffered - chunks: the rewritten name and full arguments land in the tool call's first - fragment and the arguments of its later fragments are blanked, mirroring the - text write-back. A rewrite on a stream carrying more than one distinct choice - index, or whose fragments do not line up with the rebuilt tool calls, is - reported as undeliverable, so the pipeline executor discards it and releases - the original chunks.""" post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - stream_choice_indices: Final = frozenset( - choice.index for response in responses_so_far for choice in response.choices - ) fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) - if len(stream_choice_indices) != 1: - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite( - guardrail_name, - f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written " - "back on single-choice streams", - ) if len(fragments_by_tool_call) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite @@ -1212,7 +1269,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], - guardrailed_texts: list[str], + guardrailed_texts: Sequence[str], task_mappings: list[tuple[int, int | None]], ) -> None: """ @@ -1230,12 +1287,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Override this method to customize how responses are applied to streaming responses. """ # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx) - guardrail_map: Final[dict[tuple[int, int | None], str]] = {} - for task_idx, guardrail_response in enumerate(guardrailed_texts): - mapping = task_mappings[task_idx] - choice_idx = cast(int, mapping[0]) - content_idx_optional = cast(int | None, mapping[1]) - guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response + guardrail_map: Final = MappingProxyType( + {task_mappings[index]: text for index, text in enumerate(guardrailed_texts)} + ) # Track which choices we've already set the guardrailed text for # Key: (choice_idx, content_idx), Value: boolean (True if already set) @@ -1245,34 +1299,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # choice by its index field: on n>1 streams a chunk usually carries one # choice at list position 0 whose index names the logical choice. for response in responses: - for choice in response.choices: - if isinstance(choice, litellm.StreamingChoices): - content = choice.delta.content - elif isinstance(choice, litellm.Choices): - content = choice.message.content - else: - continue - + for choice, message in ( + (choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message) + for choice in response.choices + if isinstance(choice, (StreamingChoices, Choices)) + ): + content = message.content if content is None: continue if isinstance(content, str): - # String content str_key: tuple[int, int | None] = (choice.index, None) if str_key in guardrail_map: - if str_key not in already_set: - # First chunk - set the complete guardrailed text - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = guardrail_map[str_key] - elif isinstance(choice, litellm.Choices): - choice.message.content = guardrail_map[str_key] - already_set[str_key] = True - else: - # Subsequent chunks - clear the content - if isinstance(choice, litellm.StreamingChoices): - choice.delta.content = "" - elif isinstance(choice, litellm.Choices): - choice.message.content = "" + message.content = guardrail_map[str_key] if str_key not in already_set else "" + already_set[str_key] = True elif isinstance(content, list): # List content - handle each content item diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 5d71cbe9afc..bffe6e4c123 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Literal, Optional +from typing import Any, Final, Literal, Optional import pytest @@ -1395,29 +1395,65 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert [ - (tool_call["id"], tool_call["function"]["arguments"]) - for tool_call in guardrail.seen_inputs[-1]["tool_calls"] - ] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')] + [(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]] + for inputs in guardrail.seen_inputs + ] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]] @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + @pytest.mark.parametrize("transform", [False, True]) + async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None: + from fastapi import HTTPException + from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - handler = OpenAIChatCompletionsHandler() - chunks = self._two_choice_tool_call_stream_chunks() + class ContextGuardrail(CustomGuardrail): + async def apply_guardrail( + self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], + input_type: Literal["request", "response"], logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + response: Final = request_data["response"] + assert isinstance(response, ModelResponse) + request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index) + if response.choices[0].message.content == "forbidden": + raise HTTPException(status_code=400, detail="later choice blocked") + return inputs - with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised: - await handler.process_output_streaming_response( - responses_so_far=chunks, - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, - deliver_ended_stream_rewrites=True, + chunks: Final = [ModelResponseStream(choices=[StreamingChoices( + index=index, delta=Delta(content=text, tool_calls=[{ + "index": 0, "id": f"call_{index}", "type": "function", + "function": {"name": "contact", "arguments": "{}"}, + }]), finish_reason="tool_calls", + ) for index, text in enumerate(("allowed", "forbidden"))])] + request_data: Final = {"metadata": {"trace": "retained"}} + with pytest.raises(HTTPException, match="later choice blocked"): + await OpenAIChatCompletionsHandler().process_output_streaming_response( + responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"), + request_data=request_data, deliver_ended_stream_rewrites=True, + stream_transform_sink=StreamTransformSink() if transform else None, ) + assert request_data["visited_choices"] == (0, 1) + assert request_data["metadata"]["trace"] == "retained" + assert request_data["responses"] is chunks + restored: Final = request_data["response"] + assert isinstance(restored, ModelResponse) + assert tuple(choice.index for choice in restored.choices) == (0, 1) - assert raised.value.guardrail_name == "test" - assert raised.value.reason == ( - "the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams" + @pytest.mark.asyncio + async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: + handler: Final = OpenAIChatCompletionsHandler() + chunks: Final = self._two_choice_tool_call_stream_chunks() + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, ) + arguments: Final = tuple( + "".join(call.function.arguments for chunk in chunks for choice in chunk.choices + if choice.index == index for call in choice.delta.tool_calls or ()) + for index in range(2) + ) + assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}') @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): From 12252219f3f21f0e45e92b9197d99ffd5d35d221 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:37:09 +0200 Subject: [PATCH 03/12] refactor(guardrails): separate completion choice inspection prerequisite --- .../guardrail_translation/base_translation.py | 10 +- .../chat/guardrail_translation/handler.py | 220 +++++++----------- .../test_openai_guardrail_handler.py | 70 ++---- 3 files changed, 110 insertions(+), 190 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 6280e636700..88c4dbfdce6 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional @@ -31,12 +31,8 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) - holdback_per_choice: Mapping[int, int] = field(default_factory=dict) - - def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: - self.mutated_text_per_choice = texts - self.holdback_per_choice = holdback + mutated_text_per_choice: dict[int, str] = field(default_factory=dict) + holdback_per_choice: dict[int, int] = field(default_factory=dict) @dataclass(frozen=True, slots=True) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d18fe36e3df..a12f5f164e4 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -18,11 +18,9 @@ import json import time import uuid from collections.abc import Iterator, Mapping, Sequence -from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Union, cast -from pydantic import TypeAdapter from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm @@ -48,7 +46,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( stream_item_items, unappliable_request_rewrite, ) -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import ( coerce_stream_holdback_value, ) @@ -170,8 +168,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) - guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ()) + guardrailed_texts: Final = guardrailed_inputs.get("texts", []) + guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", []) guardrailed_tools: Final = guardrailed_inputs.get("tools") if guardrailed_tools is not None: data["tools"] = ( @@ -260,12 +258,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def extract_request_tool_names(self, data: dict) -> list[str]: """Extract tool names from OpenAI chat completions request (tools[].function.name, functions[].name).""" names: Final[list[str]] = [] - for tool in data.get("tools") or (): + for tool in data.get("tools") or []: if isinstance(tool, dict) and tool.get("type") == "function": fn = tool.get("function") if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) - for fn in data.get("functions") or (): + for fn in data.get("functions") or []: if isinstance(fn, dict) and fn.get("name"): names.append(str(fn["name"])) return names @@ -389,7 +387,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict[str, object] | None = None, + request_data: dict | None = None, ) -> ModelResponse: """ Process output response by applying guardrails to text content. @@ -440,7 +438,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # create a standalone dict (SDK / direct-call path). if request_data is None: request_data = {"response": response} - request_data.setdefault("response", response) + else: + if "response" not in request_data: + request_data["response"] = response self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) @@ -460,7 +460,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) + guardrailed_texts: Final = guardrailed_inputs.get("texts", []) returned_tool_calls: Final = guardrailed_inputs.get("tool_calls") guardrailed_tool_calls: Final[list[dict[str, object]]] = ( cast(list[dict[str, object]], returned_tool_calls) @@ -494,7 +494,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None" = None, user_api_key_dict: "UserAPIKeyAuth | None" = None, - request_data: dict[str, object] | None = None, + request_data: dict | None = None, stream_transform_sink: StreamTransformSink | None = None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: @@ -550,14 +550,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict[str, object] | None, + request_data: dict | None, deliver_ended_stream_rewrites: bool = False, ) -> list["ModelResponseStream"]: """Block-only streaming path: run the guardrail so an in-flight BLOCK can terminate the stream. Text rewrites are not propagated to the client here (see ``_process_streaming_transform`` for the incremental_diff path) unless ``deliver_ended_stream_rewrites`` opts the ended-stream branch in.""" - has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far) + has_stream_ended: Final = self._first_choice_has_finished(responses_so_far) if has_stream_ended: await self._process_ended_stream( @@ -623,7 +623,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): logging_obj=litellm_logging_obj, ) - guardrailed_texts: Final = guardrailed_inputs.get("texts", ()) + guardrailed_texts: Final = guardrailed_inputs.get("texts", []) # Step 4: Apply guardrailed text back to all streaming chunks # For each choice, replace the combined text across all chunks @@ -656,36 +656,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) pre_guardrail_texts: Final = self._string_choice_contents(model_response) pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response) - inspection_responses: Final = ( - tuple( - model_response.model_copy(update=MappingProxyType({"choices": [choice]})) - for choice in model_response.choices - ) - if pre_guardrail_tool_calls and len(model_response.choices) > 1 - else (model_response,) + await self.process_output_response( + response=model_response, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=litellm_logging_obj, + user_api_key_dict=user_api_key_dict, + request_data=request_data, ) - inspection_request_data: Final = request_data if request_data is not None else {} - try: - for inspection_response in inspection_responses: - inspection_request_data["response"] = inspection_response - inspection_request_data["responses"] = ( - [ - self._narrowed_to_choice(chunk, inspection_response.choices[0].index) - for chunk in responses_so_far - ] - if len(inspection_response.choices) == 1 - else responses_so_far - ) - await self.process_output_response( - response=inspection_response, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=inspection_request_data, - ) - finally: - inspection_request_data["response"] = model_response - inspection_request_data["responses"] = responses_so_far if not deliver_ended_stream_rewrites: return await self._write_ended_stream_text_rewrites( @@ -786,7 +763,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): guardrail_to_apply: "CustomGuardrail", litellm_logging_obj: "LiteLLMLoggingObj | None", user_api_key_dict: "UserAPIKeyAuth | None", - request_data: dict[str, object] | None, + request_data: dict | None, sink: StreamTransformSink, ) -> None: """Run the guardrail over the raw accumulated text and report the @@ -798,7 +775,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """ raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far) if not raw_by_index: - sink.record(MappingProxyType({}), MappingProxyType({})) + sink.mutated_text_per_choice = {} + sink.holdback_per_choice = {} return # Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits @@ -810,72 +788,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation): indices: Final = sorted(raw_by_index.keys()) texts_to_check: Final = [raw_by_index[i] for i in indices] - resolved_request_data: Final[dict[str, object]] = ( - request_data if request_data is not None else {"responses": responses_so_far} - ) - if "responses" not in resolved_request_data: - resolved_request_data["responses"] = responses_so_far - self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict) + if request_data is None: + request_data = {"responses": responses_so_far} + elif "responses" not in request_data: + request_data["responses"] = responses_so_far + self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict) inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check) - if self._streamed_tool_call_fingerprints(responses_so_far): - assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj) - resolved_request_data["response"] = assembled - if len(assembled.choices) > 1: - choice_rounds: Final = tuple( - ( - choice, - StreamTransformSink(), - [self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far], - ) - for choice in assembled.choices - ) - try: - for choice, choice_sink, choice_chunks in choice_rounds: - resolved_request_data["response"] = assembled.model_copy( - update=MappingProxyType({"choices": [choice]}) - ) - resolved_request_data["responses"] = choice_chunks - await self._process_streaming_transform( - responses_so_far=choice_chunks, - guardrail_to_apply=guardrail_to_apply, - litellm_logging_obj=litellm_logging_obj, - user_api_key_dict=user_api_key_dict, - request_data=resolved_request_data, - sink=choice_sink, - ) - finally: - resolved_request_data["response"] = assembled - resolved_request_data["responses"] = responses_so_far - merged_texts: Final = MappingProxyType( - dict( - chain.from_iterable( - choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds - ) - ) - ) - merged_holdback: Final = MappingProxyType( - dict( - chain.from_iterable( - choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds - ) - ) - ) - sink.record(merged_texts, merged_holdback) - return - tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices) - inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python( - tuple( - MappingProxyType({"index": index, **converted}) - for index, tool_call in enumerate(tool_calls) - if (converted := self._convert_tool_call_to_dict(tool_call)) is not None - ) - ) if responses_so_far and getattr(responses_so_far[0], "model", None): inputs["model"] = responses_so_far[0].model guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail( inputs=inputs, - request_data=resolved_request_data, + request_data=request_data, input_type="response", logging_obj=litellm_logging_obj, ) @@ -895,14 +819,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): len(texts_to_check), ) - holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or () - transformed_texts: Final = MappingProxyType( - {idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)} - ) - transformed_holdback: Final = MappingProxyType( - {indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)} - ) - sink.record(transformed_texts, transformed_holdback) + holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or [] + sink.mutated_text_per_choice = { + idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts) + } + sink.holdback_per_choice = { + indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback) + } def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None: chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream)) @@ -1222,18 +1145,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _function_tool_call_fragments( responses_so_far: Sequence["ModelResponseStream"], ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: + """Group the stream's function tool-call fragments by their tool-call index, in + the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping + only the indices the builder keeps (an id and a name somewhere in the stream).""" fragments: Final = tuple( - (choice.index, tool_call) + tool_call for response in responses_so_far for choice in response.choices for tool_call in choice.delta.tool_calls or () if isinstance(tool_call, ChatCompletionDeltaToolCall) ) - identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id) - named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name) + identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) + named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) return tuple( - tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key) - for key in sorted(identified & named) + tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) ) def _write_ended_stream_tool_call_rewrites( @@ -1243,10 +1168,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation): pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...], guardrail_name: str, ) -> None: + """Write ended-stream guardrail tool-call rewrites back across the buffered + chunks: the rewritten name and full arguments land in the tool call's first + fragment and the arguments of its later fragments are blanked, mirroring the + text write-back. A rewrite on a stream carrying more than one distinct choice + index, or whose fragments do not line up with the rebuilt tool calls, is + reported as undeliverable, so the pipeline executor discards it and releases + the original chunks.""" post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) if post_guardrail_tool_calls == pre_guardrail_tool_calls: return + stream_choice_indices: Final = frozenset( + choice.index for response in responses_so_far for choice in response.choices + ) fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) + if len(stream_choice_indices) != 1: + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite + + raise UndeliverableStreamRewrite( + guardrail_name, + f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written " + "back on single-choice streams", + ) if len(fragments_by_tool_call) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite @@ -1269,7 +1212,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation): async def _apply_guardrail_responses_to_output_streaming( self, responses: list["ModelResponseStream"], - guardrailed_texts: Sequence[str], + guardrailed_texts: list[str], task_mappings: list[tuple[int, int | None]], ) -> None: """ @@ -1287,9 +1230,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Override this method to customize how responses are applied to streaming responses. """ # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx) - guardrail_map: Final = MappingProxyType( - {task_mappings[index]: text for index, text in enumerate(guardrailed_texts)} - ) + guardrail_map: Final[dict[tuple[int, int | None], str]] = {} + for task_idx, guardrail_response in enumerate(guardrailed_texts): + mapping = task_mappings[task_idx] + choice_idx = cast(int, mapping[0]) + content_idx_optional = cast(int | None, mapping[1]) + guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response # Track which choices we've already set the guardrailed text for # Key: (choice_idx, content_idx), Value: boolean (True if already set) @@ -1299,20 +1245,34 @@ class OpenAIChatCompletionsHandler(BaseTranslation): # choice by its index field: on n>1 streams a chunk usually carries one # choice at list position 0 whose index names the logical choice. for response in responses: - for choice, message in ( - (choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message) - for choice in response.choices - if isinstance(choice, (StreamingChoices, Choices)) - ): - content = message.content + for choice in response.choices: + if isinstance(choice, litellm.StreamingChoices): + content = choice.delta.content + elif isinstance(choice, litellm.Choices): + content = choice.message.content + else: + continue + if content is None: continue if isinstance(content, str): + # String content str_key: tuple[int, int | None] = (choice.index, None) if str_key in guardrail_map: - message.content = guardrail_map[str_key] if str_key not in already_set else "" - already_set[str_key] = True + if str_key not in already_set: + # First chunk - set the complete guardrailed text + if isinstance(choice, litellm.StreamingChoices): + choice.delta.content = guardrail_map[str_key] + elif isinstance(choice, litellm.Choices): + choice.message.content = guardrail_map[str_key] + already_set[str_key] = True + else: + # Subsequent chunks - clear the content + if isinstance(choice, litellm.StreamingChoices): + choice.delta.content = "" + elif isinstance(choice, litellm.Choices): + choice.message.content = "" elif isinstance(content, list): # List content - handle each content item diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index bffe6e4c123..5d71cbe9afc 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations, including tool calls. import json from collections.abc import Mapping -from typing import Any, Final, Literal, Optional +from typing import Any, Literal, Optional import pytest @@ -1395,65 +1395,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert [ - [(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]] - for inputs in guardrail.seen_inputs - ] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]] + (tool_call["id"], tool_call["function"]["arguments"]) + for tool_call in guardrail.seen_inputs[-1]["tool_calls"] + ] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')] @pytest.mark.asyncio - @pytest.mark.parametrize("transform", [False, True]) - async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None: - from fastapi import HTTPException - from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink - from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): + from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - class ContextGuardrail(CustomGuardrail): - async def apply_guardrail( - self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object], - input_type: Literal["request", "response"], logging_obj: object = None, - ) -> GenericGuardrailAPIInputs: - response: Final = request_data["response"] - assert isinstance(response, ModelResponse) - request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index) - if response.choices[0].message.content == "forbidden": - raise HTTPException(status_code=400, detail="later choice blocked") - return inputs + handler = OpenAIChatCompletionsHandler() + chunks = self._two_choice_tool_call_stream_chunks() - chunks: Final = [ModelResponseStream(choices=[StreamingChoices( - index=index, delta=Delta(content=text, tool_calls=[{ - "index": 0, "id": f"call_{index}", "type": "function", - "function": {"name": "contact", "arguments": "{}"}, - }]), finish_reason="tool_calls", - ) for index, text in enumerate(("allowed", "forbidden"))])] - request_data: Final = {"metadata": {"trace": "retained"}} - with pytest.raises(HTTPException, match="later choice blocked"): - await OpenAIChatCompletionsHandler().process_output_streaming_response( - responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"), - request_data=request_data, deliver_ended_stream_rewrites=True, - stream_transform_sink=StreamTransformSink() if transform else None, + with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised: + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, ) - assert request_data["visited_choices"] == (0, 1) - assert request_data["metadata"]["trace"] == "retained" - assert request_data["responses"] is chunks - restored: Final = request_data["response"] - assert isinstance(restored, ModelResponse) - assert tuple(choice.index for choice in restored.choices) == (0, 1) - @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None: - handler: Final = OpenAIChatCompletionsHandler() - chunks: Final = self._two_choice_tool_call_stream_chunks() - await handler.process_output_streaming_response( - responses_so_far=chunks, - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, - deliver_ended_stream_rewrites=True, + assert raised.value.guardrail_name == "test" + assert raised.value.reason == ( + "the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams" ) - arguments: Final = tuple( - "".join(call.function.arguments for chunk in chunks for choice in chunk.choices - if choice.index == index for call in choice.delta.tool_calls or ()) - for index in range(2) - ) - assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}') @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): From 68055731029c617292923b22d2764c1285d1a99d Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:41:45 +0200 Subject: [PATCH 04/12] refactor(guardrails): remove redundant validated guardrail check --- .../guardrail_hooks/unified_guardrail/unified_guardrail.py | 1 - 1 file changed, 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 9134a1cd0f3..a33b7fd3826 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -1402,7 +1402,6 @@ class UnifiedLLMGuardrails(CustomLogger): chunks_yielded and not verdict_settled and translation_class is not None - and isinstance(resolved_guardrail, CustomGuardrail) ): await self._scan_released_stream_after_disconnect( endpoint_translation=translation_class(), From 9901032e4cbbfa99262dca7ff622365c48ad576d Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:41:49 +0200 Subject: [PATCH 05/12] style(guardrails): format disconnect guard condition --- .../guardrail_hooks/unified_guardrail/unified_guardrail.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index a33b7fd3826..786e90c6b28 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -1398,11 +1398,7 @@ class UnifiedLLMGuardrails(CustomLogger): yield error_item except (GeneratorExit, asyncio.CancelledError): translation_class: Final = None if call_type is None else mappings.get(CallTypes(call_type)) - if ( - chunks_yielded - and not verdict_settled - and translation_class is not None - ): + if chunks_yielded and not verdict_settled and translation_class is not None: await self._scan_released_stream_after_disconnect( endpoint_translation=translation_class(), responses_released=responses_yielded, From c99cc59ddaeb29f0f30ec0693b9314e67b315e6c Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 09:43:10 +0200 Subject: [PATCH 06/12] fix(guardrails): preserve the existing stream sink value types --- .../base_llm/guardrail_translation/base_translation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 6280e636700..f3dd67a2bb9 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -31,12 +31,12 @@ class StreamTransformSink: single call, not shared state. """ - mutated_text_per_choice: Mapping[int, str] = field(default_factory=dict) - holdback_per_choice: Mapping[int, int] = field(default_factory=dict) + mutated_text_per_choice: dict[int, str] = field(default_factory=dict) + holdback_per_choice: dict[int, int] = field(default_factory=dict) def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None: - self.mutated_text_per_choice = texts - self.holdback_per_choice = holdback + self.mutated_text_per_choice = dict(texts) + self.holdback_per_choice = dict(holdback) @dataclass(frozen=True, slots=True) From b2b61d21999dca4fa086d7b19f5fd41f5895b990 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:00:45 +0200 Subject: [PATCH 07/12] test(guardrails): verify disconnect inspection retains tool context --- .../unified_guardrails/test_unified_guardrail.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py index 3efbe6475b2..9a4fa3ba91b 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py @@ -2706,7 +2706,12 @@ class TestStreamingClientDisconnectScan: assert [call.function.name for call in released[0].choices[0].delta.tool_calls] == ["get_weather"] assert guardrail.received_texts == [["BLOCKME"]], guardrail.received_texts - assert guardrail.received_tool_calls == [], guardrail.received_tool_calls + assert guardrail.received_tool_calls == [[{ + "id": "call_1", + "type": "function", + "index": 0, + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + }]] @pytest.mark.asyncio async def test_closing_after_a_released_tool_call_under_incremental_diff_scans_no_held_back_text(self): From 37d6a1cacd3493ffdf50430a2896672776fa105b Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:03:34 +0200 Subject: [PATCH 08/12] refactor(guardrails): type the shared constructor extension fields --- litellm/integrations/custom_guardrail.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 45b20211ac5..583e9108af7 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,8 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, - **kwargs, - ): + **kwargs: object, + ) -> None: """ Initialize the CustomGuardrail class @@ -303,7 +303,7 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages - stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config + stream_scope_arg: Final[object] = kwargs.pop("stream_scope", None) self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None self.logging_only_continue_on_input_failure = False From 3b3fc05d5cc04cba068dc3122659881e7272afdc Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:03:36 +0200 Subject: [PATCH 09/12] refactor(guardrails): collect checked stream tails outside the replay loop --- .../unified_guardrail/unified_guardrail.py | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 786e90c6b28..7b1d646bdec 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -809,19 +809,17 @@ class UnifiedLLMGuardrails(CustomLogger): if self._chunk_has_tool_calls(buffered_item) ) - async def checked_tail() -> AsyncGenerator[object, None]: - try: - async for tail_chunk in self._emit_stream_tail( + tail_chunks: Final = tuple( + [ + chunk + async for chunk in self._checked_stream_tail( last_chunk=last_chunk, final_round=_round, responses_so_far=responses_so_far, responses_yielded=responses_yielded, - ): - yield tail_chunk - except _StreamTerminated as exc: - yield exc - - tail_chunks: Final = tuple([chunk async for chunk in checked_tail()]) + ) + ] + ) if tail_chunks and isinstance(tail_chunks[-1], _StreamTerminated): for error_chunk in tail_chunks[:-1]: yield error_chunk @@ -843,6 +841,25 @@ class UnifiedLLMGuardrails(CustomLogger): except _StreamTerminated: return + async def _checked_stream_tail( + self, + *, + last_chunk: object | None, + final_round: Callable[[object, bool], AsyncGenerator[object, None]], + responses_so_far: Sequence[object], + responses_yielded: list[object], + ) -> AsyncGenerator[object, None]: + try: + async for chunk in self._emit_stream_tail( + last_chunk=last_chunk, + final_round=final_round, + responses_so_far=responses_so_far, + responses_yielded=responses_yielded, + ): + yield chunk + except _StreamTerminated as exc: + yield exc + async def _emit_stream_tail( self, *, From a07ec080365c633968a15c0198f6afac6c4683d1 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:09:17 +0200 Subject: [PATCH 10/12] fix(guardrails): forward typed message logging flags to the logger --- litellm/integrations/custom_guardrail.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 583e9108af7..f7457dd4ff9 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,6 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, + turn_off_message_logging: bool = False, + message_logging: bool = True, **kwargs: object, ) -> None: """ @@ -323,7 +325,11 @@ class CustomGuardrail(CustomLogger): "(default) to enforce validation and fail at startup.", validation_error, ) - super().__init__(**kwargs) + super().__init__( + turn_off_message_logging=turn_off_message_logging, + message_logging=message_logging, + **kwargs, + ) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: """Return a custom violation message if template is configured.""" From fa10963e3d3fd03020c07a35d9c5d09bae3a7b80 Mon Sep 17 00:00:00 2001 From: albertbausili Date: Fri, 16 Oct 2026 10:11:45 +0200 Subject: [PATCH 11/12] refactor(guardrails): keep legacy constructor typing outside choice inspection --- litellm/integrations/custom_guardrail.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index f7457dd4ff9..45b20211ac5 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -256,10 +256,8 @@ class CustomGuardrail(CustomLogger): scan_raw_request: bool = False, only_scan_new_messages: bool = False, timeout: float | None = None, - turn_off_message_logging: bool = False, - message_logging: bool = True, - **kwargs: object, - ) -> None: + **kwargs, + ): """ Initialize the CustomGuardrail class @@ -305,7 +303,7 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages - stream_scope_arg: Final[object] = kwargs.pop("stream_scope", None) + stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None self.logging_only_continue_on_input_failure = False @@ -325,11 +323,7 @@ class CustomGuardrail(CustomLogger): "(default) to enforce validation and fail at startup.", validation_error, ) - super().__init__( - turn_off_message_logging=turn_off_message_logging, - message_logging=message_logging, - **kwargs, - ) + super().__init__(**kwargs) def render_violation_message(self, default: str, context: Mapping[str, object] | None = None) -> str: """Return a custom violation message if template is configured.""" From 296681bdc330a7d0656d75a48400d2aa02aab416 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Wed, 21 Oct 2026 23:26:39 +0200 Subject: [PATCH 12/12] fix(guardrails): pass the resolved guardrail to the end-of-stream scan check --- .../guardrail_hooks/unified_guardrail/unified_guardrail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index 4a985234a6c..f86d718bd86 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -1385,7 +1385,7 @@ class UnifiedLLMGuardrails(CustomLogger): end_scan_key: Final = endpoint_translation.get_streaming_scan_key(responses_so_far) verdict_settled = True if _is_redundant_scan(end_scan_key, last_scan_key) and not _owes_ended_scan( - endpoint_translation, guardrail_to_apply, responses_so_far, last_scan_key + endpoint_translation, resolved_guardrail, responses_so_far, last_scan_key ): verbose_proxy_logger.debug( "Skipping end-of-stream scan for guardrail %s: the last sampled round already scanned it all",