This commit is contained in:
AlbertNeuralTrust 2026-11-01 22:53:49 +01:00 • committed by GitHub
commit d4467763b3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 855 additions and 134 deletions

View file

@ -1532,6 +1532,9 @@ class CustomGuardrail(CustomLogger):
end_time=end_time,
duration=duration,
masked_entity_count=masked_entity_count,
# Set by callers that inspect one choice at a time, so the per-choice entries of an
# n > 1 response can be told apart.
choice_index=request_data.get("guardrail_choice_index"),
**(tracing_detail or {}),
)

View file

@ -141,6 +141,7 @@ class GenAIMapper:
LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score,
LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score,
LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count,
LiteLLM.GUARDRAIL_CHOICE_INDEX: lambda d: d.choice_index,
LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration,
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,

View file

@ -253,6 +253,7 @@ class GuardrailSpanData:
mode: str | None = None
status: str | None = None
masked_entity_count: int | None = None
choice_index: int | None = None
provider: str | None = None
action: str | None = None
# The guardrail verdict / provider response (e.g. the moderation result),
@ -313,6 +314,7 @@ class GuardrailSpanData:
mode=_guardrail_mode_str(get("guardrail_mode")),
status=status,
masked_entity_count=_total_masked_entities(get("masked_entity_count")),
choice_index=as_int(get("choice_index")),
provider=as_str(get("guardrail_provider")),
action=as_str(get("guardrail_action")),
response_json=_json_or_none(response) if response is not None else None,

View file

@ -317,6 +317,7 @@ class LiteLLM:
GUARDRAIL_CONFIDENCE_SCORE: Final = "litellm.guardrail.confidence_score"
GUARDRAIL_RISK_SCORE: Final = "litellm.guardrail.risk_score"
GUARDRAIL_MASKED_ENTITY_COUNT: Final = "litellm.guardrail.masked_entity_count"
GUARDRAIL_CHOICE_INDEX: Final = "litellm.guardrail.choice_index"
GUARDRAIL_DURATION: Final = "litellm.guardrail.duration"
GUARDRAIL_ID: Final = "litellm.guardrail.id"
GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"

View file

@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
@ -34,6 +34,10 @@ class StreamTransformSink:
mutated_text_per_choice: dict[int, str] = field(default_factory=dict)
holdback_per_choice: dict[int, int] = field(default_factory=dict)
def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None:
self.mutated_text_per_choice = dict(texts)
self.holdback_per_choice = dict(holdback)
@dataclass(frozen=True, slots=True)
class StreamingScanKey:

View file

@ -18,9 +18,11 @@ import json
import time
import uuid
from collections.abc import Iterator, Mapping, Sequence
from itertools import chain, groupby
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Union, cast
from pydantic import TypeAdapter
from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
@ -50,7 +52,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
stream_item_items,
unappliable_request_rewrite,
)
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
coerce_stream_holdback_value,
)
@ -172,8 +174,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", [])
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ())
guardrailed_tools: Final = guardrailed_inputs.get("tools")
if guardrailed_tools is not None:
data["tools"] = (
@ -392,7 +394,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict | None = None,
request_data: dict[str, object] | None = None,
) -> ModelResponse:
"""
Process output response by applying guardrails to text content.
@ -443,9 +445,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# create a standalone dict (SDK / direct-call path).
if request_data is None:
request_data = {"response": response}
else:
if "response" not in request_data:
request_data["response"] = response
request_data.setdefault("response", response)
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
@ -465,7 +465,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
logging_obj=litellm_logging_obj,
)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
cast(list[dict[str, object]], returned_tool_calls)
@ -499,7 +499,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
user_api_key_dict: "UserAPIKeyAuth | None" = None,
request_data: dict | None = None,
request_data: dict[str, object] | None = None,
stream_transform_sink: StreamTransformSink | None = None,
deliver_ended_stream_rewrites: bool = False,
) -> list["ModelResponseStream"]:
@ -555,14 +555,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None",
user_api_key_dict: "UserAPIKeyAuth | None",
request_data: dict | None,
request_data: dict[str, object] | None,
deliver_ended_stream_rewrites: bool = False,
) -> list["ModelResponseStream"]:
"""Block-only streaming path: run the guardrail so an in-flight BLOCK can
terminate the stream. Text rewrites are not propagated to the client here
(see ``_process_streaming_transform`` for the incremental_diff path) unless
``deliver_ended_stream_rewrites`` opts the ended-stream branch in."""
has_stream_ended: Final = self._first_choice_has_finished(responses_so_far)
has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far)
if has_stream_ended:
await self._process_ended_stream(
@ -609,8 +609,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# create a standalone dict (SDK / direct-call path).
if request_data is None:
request_data = {"responses": responses_so_far}
owes_responses = False # rebind-ok: the standalone dict is ours to keep
else:
if "responses" not in request_data:
owes_responses = "responses" not in request_data # rebind-ok: tracks the key this scan adds
if owes_responses:
request_data["responses"] = responses_so_far
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
@ -621,14 +623,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# Include model information from the first response if available
if responses_so_far and hasattr(responses_so_far[0], "model") and responses_so_far[0].model:
inputs["model"] = responses_so_far[0].model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
input_type="response",
logging_obj=litellm_logging_obj,
)
try:
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
input_type="response",
logging_obj=litellm_logging_obj,
)
finally:
if owes_responses:
request_data.pop("responses", None)
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
# Step 4: Apply guardrailed text back to all streaming chunks
# For each choice, replace the combined text across all chunks
@ -661,13 +667,63 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
pre_guardrail_texts: Final = self._string_choice_contents(model_response)
pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response)
await self.process_output_response(
response=model_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=request_data,
inspection_responses: Final = (
tuple(
model_response.model_copy(update=MappingProxyType({"choices": [choice]}))
for choice in model_response.choices
)
if pre_guardrail_tool_calls and len(model_response.choices) > 1
else (model_response,)
)
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
inspection_request_data: Final = request_data if request_data is not None else {}
missing: Final = object()
prior_response: Final = inspection_request_data.get("response", missing)
prior_responses: Final = inspection_request_data.get("responses", missing)
prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing)
inspection_request_data.pop("guardrail_choice_index", None)
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
try:
for inspection_response in inspection_responses:
inspection_request_data["response"] = inspection_response
inspection_request_data["guardrail_choice_index"] = (
inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None
)
inspection_request_data["responses"] = (
[
self._narrowed_to_choice(chunk, inspection_response.choices[0].index)
for chunk in responses_so_far
]
if len(inspection_response.choices) == 1
else responses_so_far
)
try:
await self.process_output_response(
response=inspection_response,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=inspection_request_data,
)
except UndeliverableStreamRewrite as exc:
if undeliverable is None:
undeliverable = exc
finally:
if prior_response is missing:
inspection_request_data.pop("response", None)
else:
inspection_request_data["response"] = prior_response
if prior_responses is missing:
inspection_request_data.pop("responses", None)
else:
inspection_request_data["responses"] = prior_responses
if prior_choice_index is missing:
inspection_request_data.pop("guardrail_choice_index", None)
else:
inspection_request_data["guardrail_choice_index"] = prior_choice_index
if undeliverable is not None:
raise undeliverable
if not deliver_ended_stream_rewrites:
return
await self._write_ended_stream_text_rewrites(
@ -682,6 +738,95 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
)
@staticmethod
def _streamed_argument_fragments(
responses_so_far: Sequence["ModelResponseStream"],
) -> Iterator[tuple[tuple[int, int], str]]:
for chunk in responses_so_far:
for choice in chunk.choices:
for call in choice.delta.tool_calls or ():
arguments = getattr(getattr(call, "function", None), "arguments", None)
yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else ""
@staticmethod
def _parses_as_json(arguments: str) -> bool:
if not arguments:
return False
try:
json.loads(arguments)
except ValueError:
return False
return True
@staticmethod
def _joined_streamed_arguments(
responses_so_far: Sequence["ModelResponseStream"],
) -> Mapping[tuple[int, int], str]:
"""The argument text streamed so far per (choice index, tool call index).
Grouped in one pass. Rescanning every fragment once per distinct call made a sampled
round cost the number of calls times the length of the stream, which at n=2 grows with
the square of the streamed arguments.
"""
fragments: Final = sorted(
OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far),
key=lambda fragment: fragment[0],
)
return MappingProxyType(
{call: "".join(text for _, text in group) for call, group in groupby(fragments, key=lambda f: f[0])}
)
@staticmethod
def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool:
"""Whether every streamed tool call now carries arguments that parse as JSON.
A call that has announced itself but emitted no argument text yet is still in flight, so a
parallel call finishing first does not make the whole set complete.
"""
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
if not joined:
return False
return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values())
@staticmethod
def _streamed_call_ids(responses_so_far: Sequence["ModelResponseStream"]) -> Mapping[tuple[int, int], str]:
"""The id each (choice index, tool call index) announced itself with."""
choices: Final = chain.from_iterable(chunk.choices for chunk in responses_so_far)
calls: Final = chain.from_iterable(
((choice.index, call) for call in choice.delta.tool_calls or ()) for choice in choices
)
return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id})
@staticmethod
def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices":
kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished]
message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None}))
return choice.model_copy(update=MappingProxyType({"message": message}))
@staticmethod
def _without_unfinished_tool_calls(
assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"]
) -> "ModelResponse":
"""Copy of the rebuilt response without the calls still in flight when the stream ended.
Rebuilding gives a call that never streamed complete arguments an empty JSON object, which
reads to a guardrail as a real call taking no arguments rather than an unfinished one. The
calls are matched by the id they announced, since the rebuilt order need not follow the
order the stream gave them.
"""
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
unfinished: Final = frozenset(
call_id
for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items()
if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, ""))
)
if not unfinished:
return assembled
pruned: Final = [
OpenAIChatCompletionsHandler._choice_without(choice, unfinished) for choice in assembled.choices
]
return assembled.model_copy(update=MappingProxyType({"choices": pruned}))
@staticmethod
def _rebuild_ended_stream_per_choice(
responses_so_far: Sequence["ModelResponseStream"],
@ -706,6 +851,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
)
for index in choice_indices
)
if not rebuilt_by_index:
whole: Final = stream_chunk_builder(chunks=list(responses_so_far), logging_obj=litellm_logging_obj)
return cast(ModelResponse, whole) if whole is not None else ModelResponse(choices=[])
(_, base_response), *_ = rebuilt_by_index
stitched_choices: Final = [
rebuilt.choices[0].model_copy(update=MappingProxyType({"index": index}))
@ -763,7 +911,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: "LiteLLMLoggingObj | None",
user_api_key_dict: "UserAPIKeyAuth | None",
request_data: dict | None,
request_data: dict[str, object] | None,
sink: StreamTransformSink,
) -> None:
"""Run the guardrail over the raw accumulated text and report the
@ -775,8 +923,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
"""
raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far)
if not raw_by_index:
sink.mutated_text_per_choice = {}
sink.holdback_per_choice = {}
sink.record(MappingProxyType({}), MappingProxyType({}))
return
# Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits
@ -788,44 +935,150 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
indices: Final = sorted(raw_by_index.keys())
texts_to_check: Final = [raw_by_index[i] for i in indices]
if request_data is None:
request_data = {"responses": responses_so_far}
elif "responses" not in request_data:
request_data["responses"] = responses_so_far
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if responses_so_far and getattr(responses_so_far[0], "model", None):
inputs["model"] = responses_so_far[0].model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=request_data,
input_type="response",
logging_obj=litellm_logging_obj,
resolved_request_data: Final[dict[str, object]] = (
request_data if request_data is not None else {"responses": responses_so_far}
)
missing: Final = object()
prior_response: Final = resolved_request_data.get("response", missing)
prior_responses: Final = resolved_request_data.get("responses", missing)
prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing)
resolved_request_data.pop("guardrail_choice_index", None)
if len(indices) == 1:
resolved_request_data["guardrail_choice_index"] = indices[0]
if "responses" not in resolved_request_data:
resolved_request_data["responses"] = responses_so_far
self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict)
try:
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
if self._streamed_tool_call_fingerprints(responses_so_far):
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
assembled: Final = (
None
if not stream_ended and not tool_arguments_complete
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
)
inspected: Final = (
None
if assembled is None
else assembled
if tool_arguments_complete
else self._without_unfinished_tool_calls(assembled, responses_so_far)
)
if inspected is not None:
resolved_request_data["response"] = inspected
if inspected is not None and len(inspected.choices) > 1:
choice_rounds: Final = tuple(
(
choice,
StreamTransformSink(),
[self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far],
)
for choice in inspected.choices
)
from litellm.proxy.policy_engine.pipeline_executor import (
UndeliverableStreamRewrite,
)
returned_texts = guardrailed_inputs.get("texts")
# No "texts" key means the guardrail made no change (action NONE): the raw
# accumulated text is the guardrailed text. A present-but-shorter list is a
# guardrail contract violation; those choices are omitted below (withheld,
# not emitted raw) so a malformed response fails closed instead of leaking.
if returned_texts is None:
returned_texts = texts_to_check
elif len(returned_texts) < len(texts_to_check):
verbose_proxy_logger.warning(
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
"streaming transform path; withholding the unmatched choices to fail closed.",
len(returned_texts),
len(texts_to_check),
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: first failure
try:
for choice, choice_sink, choice_chunks in choice_rounds:
resolved_request_data["response"] = inspected.model_copy(
update=MappingProxyType({"choices": [choice]})
)
resolved_request_data["responses"] = choice_chunks
resolved_request_data["guardrail_choice_index"] = choice.index
try:
await self._process_streaming_transform(
responses_so_far=choice_chunks,
guardrail_to_apply=guardrail_to_apply,
litellm_logging_obj=litellm_logging_obj,
user_api_key_dict=user_api_key_dict,
request_data=resolved_request_data,
sink=choice_sink,
)
except UndeliverableStreamRewrite as exc:
if transform_undeliverable is None:
transform_undeliverable = exc
finally:
resolved_request_data["response"] = assembled
resolved_request_data["responses"] = responses_so_far
resolved_request_data.pop("guardrail_choice_index", None)
if transform_undeliverable is not None:
raise transform_undeliverable
merged_texts: Final = MappingProxyType(
dict(
chain.from_iterable(
choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds
)
)
)
merged_holdback: Final = MappingProxyType(
dict(
chain.from_iterable(
choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds
)
)
)
sink.record(merged_texts, merged_holdback)
return
if assembled is not None and tool_arguments_complete:
tool_calls: Final = chain.from_iterable(
choice.message.tool_calls or () for choice in assembled.choices
)
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
tuple(
MappingProxyType({"index": index, **converted})
for index, tool_call in enumerate(tool_calls)
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
)
)
if responses_so_far and getattr(responses_so_far[0], "model", None):
inputs["model"] = responses_so_far[0].model
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
inputs=inputs,
request_data=resolved_request_data,
input_type="response",
logging_obj=litellm_logging_obj,
)
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or []
sink.mutated_text_per_choice = {
idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)
}
sink.holdback_per_choice = {
indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)
}
returned_texts = guardrailed_inputs.get("texts")
# No "texts" key means the guardrail made no change (action NONE): the raw
# accumulated text is the guardrailed text. A present-but-shorter list is a
# guardrail contract violation; those choices are omitted below (withheld,
# not emitted raw) so a malformed response fails closed instead of leaking.
if returned_texts is None:
returned_texts = texts_to_check
elif len(returned_texts) < len(texts_to_check):
verbose_proxy_logger.warning(
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
"streaming transform path; withholding the unmatched choices to fail closed.",
len(returned_texts),
len(texts_to_check),
)
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or ()
transformed_texts: Final = MappingProxyType(
{idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)}
)
transformed_holdback: Final = MappingProxyType(
{
indices[i]: coerce_stream_holdback_value(holdback[i])
for i in range(len(indices))
if i < len(holdback)
}
)
sink.record(transformed_texts, transformed_holdback)
finally:
for key, prior in (
("response", prior_response),
("responses", prior_responses),
("guardrail_choice_index", prior_choice_index),
):
if prior is missing:
resolved_request_data.pop(key, None)
else:
resolved_request_data[key] = prior
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream))
@ -1142,20 +1395,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
def _function_tool_call_fragments(
responses_so_far: Sequence["ModelResponseStream"],
) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]:
"""Group the stream's function tool-call fragments by their tool-call index, in
the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping
only the indices the builder keeps (an id and a name somewhere in the stream)."""
fragments: Final = tuple(
tool_call
(choice.index, tool_call)
for response in responses_so_far
for choice in response.choices
for tool_call in choice.delta.tool_calls or ()
if isinstance(tool_call, ChatCompletionDeltaToolCall)
)
identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id)
named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name)
identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id)
named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name)
return tuple(
tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named)
tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key)
for key in sorted(identified & named)
)
def _write_ended_stream_tool_call_rewrites(
@ -1165,28 +1416,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...],
guardrail_name: str,
) -> None:
"""Write ended-stream guardrail tool-call rewrites back across the buffered
chunks: the rewritten name and full arguments land in the tool call's first
fragment and the arguments of its later fragments are blanked, mirroring the
text write-back. A rewrite on a stream carrying more than one distinct choice
index, or whose fragments do not line up with the rebuilt tool calls, is
reported as undeliverable, so the pipeline executor discards it and releases
the original chunks."""
post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response)
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
return
stream_choice_indices: Final = frozenset(
choice.index for response in responses_so_far for choice in response.choices
)
fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far)
if len(stream_choice_indices) != 1:
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
raise UndeliverableStreamRewrite(
guardrail_name,
f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written "
"back on single-choice streams",
)
if len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
@ -1209,7 +1442,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
async def _apply_guardrail_responses_to_output_streaming(
self,
responses: list["ModelResponseStream"],
guardrailed_texts: list[str],
guardrailed_texts: Sequence[str],
task_mappings: list[tuple[int, int | None]],
) -> None:
"""
@ -1227,12 +1460,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
Override this method to customize how responses are applied to streaming responses.
"""
# Build a mapping of what guardrailed text to use for each (choice_idx, content_idx)
guardrail_map: Final[dict[tuple[int, int | None], str]] = {}
for task_idx, guardrail_response in enumerate(guardrailed_texts):
mapping = task_mappings[task_idx]
choice_idx = cast(int, mapping[0])
content_idx_optional = cast(int | None, mapping[1])
guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response
guardrail_map: Final = MappingProxyType(
{task_mappings[index]: text for index, text in enumerate(guardrailed_texts)}
)
# Track which choices we've already set the guardrailed text for
# Key: (choice_idx, content_idx), Value: boolean (True if already set)
@ -1242,34 +1472,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
# choice by its index field: on n>1 streams a chunk usually carries one
# choice at list position 0 whose index names the logical choice.
for response in responses:
for choice in response.choices:
if isinstance(choice, litellm.StreamingChoices):
content = choice.delta.content
elif isinstance(choice, litellm.Choices):
content = choice.message.content
else:
continue
for choice, message in (
(choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message)
for choice in response.choices
if isinstance(choice, (StreamingChoices, Choices))
):
content = message.content
if content is None:
continue
if isinstance(content, str):
# String content
str_key: tuple[int, int | None] = (choice.index, None)
if str_key in guardrail_map:
if str_key not in already_set:
# First chunk - set the complete guardrailed text
if isinstance(choice, litellm.StreamingChoices):
choice.delta.content = guardrail_map[str_key]
elif isinstance(choice, litellm.Choices):
choice.message.content = guardrail_map[str_key]
already_set[str_key] = True
else:
# Subsequent chunks - clear the content
if isinstance(choice, litellm.StreamingChoices):
choice.delta.content = ""
elif isinstance(choice, litellm.Choices):
choice.message.content = ""
message.content = guardrail_map[str_key] if str_key not in already_set else ""
already_set[str_key] = True
elif isinstance(content, list):
# List content - handle each content item

View file

@ -3591,6 +3591,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
"end_time",
"duration",
"masked_entity_count",
"choice_index",
"guardrail_id",
"policy_template",
"detection_method",
@ -3631,6 +3632,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
}
"""
choice_index: ReadOnly[int | None]
"""Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice."""
guardrail_id: str | None
"""Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'"""

View file

@ -344,6 +344,19 @@ def test_guardrail_block_span_is_error_and_carries_verdict():
assert a[LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT] == 2
def test_guardrail_span_carries_the_choice_index():
"""A per-choice guardrail entry must say on the span which choice it inspected."""
engine, exporter = _engine()
engine.emit(
SpanRole.GUARDRAIL,
GuardrailSpanData.from_logging_entry(
{"guardrail_name": "g", "guardrail_status": "success", "choice_index": 1}
),
)
(span,) = exporter.get_finished_spans()
assert span.attributes[LiteLLM.GUARDRAIL_CHOICE_INDEX] == 1
def test_guardrail_success_span_is_unset():
"""On success the status is left UNSET (semconv default) — not forced OK."""
engine, exporter = _engine()

View file

@ -7,7 +7,7 @@ with guardrail transformations, including tool calls.
import json
from collections.abc import Mapping
from typing import Any, Literal, Optional
from typing import Any, Final, Literal, Optional
import pytest
@ -1395,29 +1395,501 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
)
assert [
(tool_call["id"], tool_call["function"]["arguments"])
for tool_call in guardrail.seen_inputs[-1]["tool_calls"]
] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')]
[(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]]
for inputs in guardrail.seen_inputs
] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]]
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self):
@pytest.mark.parametrize("transform", [False, True])
async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None:
from fastapi import HTTPException
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class ContextGuardrail(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index)
if response.choices[0].message.content == "forbidden":
raise HTTPException(status_code=400, detail="later choice blocked")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("allowed", "forbidden"))])]
request_data: Final = {"metadata": {"trace": "retained"}}
with pytest.raises(HTTPException, match="later choice blocked"):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"),
request_data=request_data, deliver_ended_stream_rewrites=True,
stream_transform_sink=StreamTransformSink() if transform else None,
)
assert request_data["visited_choices"] == (0, 1)
assert request_data["metadata"]["trace"] == "retained"
assert "response" not in request_data
assert "responses" not in request_data
@pytest.mark.asyncio
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
"""An undeliverable rewrite on one choice must not leave later choices uninspected.
The caller releases the original chunks when it catches UndeliverableStreamRewrite, so
bailing out of the loop would let a request with n > 1 pair a redactable choice with a
prohibited one and have the whole response delivered unchecked.
"""
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
handler = OpenAIChatCompletionsHandler()
chunks = self._two_choice_tool_call_stream_chunks()
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised:
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None:
"""The transform path runs its own per-choice loop and needs the same rule."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class UndeliverableOnFirstChoice(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
response: Final = request_data["response"]
assert isinstance(response, ModelResponse)
index: Final = response.choices[0].index
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
if index == 0:
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
return inputs
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("redacted", "prohibited"))])]
request_data: Final = {}
with pytest.raises(UndeliverableStreamRewrite):
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
request_data=request_data, deliver_ended_stream_rewrites=True,
stream_transform_sink=StreamTransformSink(),
)
assert request_data["visited_choices"] == (0, 1)
@pytest.mark.asyncio
async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None:
"""A buffered stream has no finish_reason yet, so the ended path must key off the opt-in.
Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered
caller on the in-flight path, where rewrites are never written back.
"""
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class PathProbe(CustomGuardrail):
saw_assembled_response: bool = False
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse)
return inputs
chunks: Final = [
ModelResponseStream(
choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)]
)
]
assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=PathProbe(guardrail_name="probe"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
assert PathProbe.saw_assembled_response is True
@pytest.mark.asyncio
async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None:
"""The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks.
Reporting an empty holdback here would release text the guardrail asked to withhold.
"""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class HoldingGuardrail(CustomGuardrail):
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
texts: Final = inputs.get("texts") or ()
return {**inputs, "stream_holdback_chars": [len(text) for text in texts]}
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=index, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "contact", "arguments": "{}"},
}]), finish_reason="tool_calls",
) for index, text in enumerate(("alpha", "bravo-long"))])]
sink: Final = StreamTransformSink()
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"),
litellm_logging_obj=None,
stream_transform_sink=sink,
)
assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")}
@pytest.mark.asyncio
async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None:
"""Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent.
Sending them would hand the guardrail something like `{"ci`, which main never did.
"""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(inputs.get("tool_calls"))
return inputs
def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream:
return ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content=text, tool_calls=[{
"index": 0, "id": "call_0", "type": "function",
"function": {"name": "weather", "arguments": arguments},
}]), finish_reason=finish_reason,
)])
RecordingGuardrail.seen = []
handler: Final = OpenAIChatCompletionsHandler()
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
partial: Final = [chunk("looking", '{"ci', None)]
await handler.process_output_streaming_response(
responses_so_far=partial, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert RecordingGuardrail.seen == [None]
RecordingGuardrail.seen = []
complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")]
await handler.process_output_streaming_response(
responses_so_far=complete, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
sent: Final = RecordingGuardrail.seen[-1]
assert sent is not None
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
@pytest.mark.asyncio
async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None:
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(inputs.get("tool_calls"))
return inputs
def call(index: int, arguments: str) -> dict[str, object]:
return {
"index": index, "id": f"call_{index}", "type": "function",
"function": {"name": "weather", "arguments": arguments},
}
RecordingGuardrail.seen = []
handler: Final = OpenAIChatCompletionsHandler()
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
finish_reason=None,
)])]
await handler.process_output_streaming_response(
responses_so_far=one_still_empty, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
assert RecordingGuardrail.seen == [None]
RecordingGuardrail.seen = []
both_complete: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0,
delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]),
finish_reason="tool_calls",
)])]
await handler.process_output_streaming_response(
responses_so_far=both_complete, guardrail_to_apply=guardrail,
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
)
sent: Final = RecordingGuardrail.seen[-1]
assert sent is not None
assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}']
@pytest.mark.asyncio
async def test_a_buffered_stream_that_carried_no_choices_is_not_an_error(self) -> None:
"""A stream can end carrying only usage, and a policy step still runs over it."""
from litellm.types.utils import ModelResponseStream
chunks: Final = [ModelResponseStream(choices=[], usage={"prompt_tokens": 3, "completion_tokens": 0,
"total_tokens": 3})]
result: Final = await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
request_data={},
deliver_ended_stream_rewrites=True,
)
assert result is chunks
@pytest.mark.asyncio
async def test_a_round_with_nothing_to_judge_costs_one_call_at_n2(self) -> None:
"""Per-choice rounds buy tool-call context. While no call is complete there is none to buy,
and charging a vendor once per choice for every sampled round buys nothing."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class CountingGuardrail(CustomGuardrail):
calls: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).calls.append(inputs.get("texts"))
return inputs
def choice(index: int, arguments: str) -> StreamingChoices:
return StreamingChoices(
index=index,
delta=Delta(content="looking", tool_calls=[
{"index": 0, "id": f"call_{index}", "type": "function",
"function": {"name": "weather", "arguments": arguments}},
]),
finish_reason=None,
)
assert raised.value.guardrail_name == "test"
assert raised.value.reason == (
"the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams"
CountingGuardrail.calls = []
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=[ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"ci')])],
guardrail_to_apply=CountingGuardrail(guardrail_name="counting"),
litellm_logging_obj=None, request_data={},
stream_transform_sink=StreamTransformSink(),
)
assert len(CountingGuardrail.calls) == 1
@pytest.mark.asyncio
async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None:
"""Half a JSON object is not a call a guardrail can judge, at any number of choices."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(request_data.get("response"))
return inputs
def choice(index: int, arguments: str, label: str) -> StreamingChoices:
return StreamingChoices(
index=index,
delta=Delta(content="looking", tool_calls=[
{"index": 0, "id": "call_" + label, "type": "function",
"function": {"name": "weather", "arguments": arguments}},
]),
finish_reason="tool_calls",
)
RecordingGuardrail.seen = []
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=[ModelResponseStream(choices=[
choice(0, '{"city": "Pa', "partial"), choice(1, '{"city": "Rome"}', "rome"),
])],
guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"),
litellm_logging_obj=None, request_data={},
stream_transform_sink=StreamTransformSink(),
)
arguments: Final = tuple(
tool_call.function.arguments
for response in RecordingGuardrail.seen
for single in getattr(response, "choices", None) or ()
for tool_call in single.message.tool_calls or ()
)
assert arguments == ('{"city": "Rome"}',)
@pytest.mark.asyncio
async def test_a_caller_cannot_set_the_choice_index_label(self) -> None:
"""The label says which choice the round inspected, so only the round may set it."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class LabelReadingGuardrail(CustomGuardrail):
seen: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen.append(request_data.get("guardrail_choice_index"))
return inputs
LabelReadingGuardrail.seen = []
request_data: Final[dict[str, object]] = {"model": "gpt-4", "guardrail_choice_index": 7}
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0,
delta=Delta(content="looking", tool_calls=[
{"index": 0, "id": "call_0", "type": "function",
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
]),
finish_reason="tool_calls",
)])]
await OpenAIChatCompletionsHandler().process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=LabelReadingGuardrail(guardrail_name="label"),
litellm_logging_obj=None,
request_data=request_data,
stream_transform_sink=StreamTransformSink(),
)
assert LabelReadingGuardrail.seen == [0]
assert request_data["guardrail_choice_index"] == 7
@pytest.mark.asyncio
async def test_a_transform_round_leaves_the_callers_request_data_as_it_found_it(self) -> None:
"""The round owns these keys only while it runs; what it leaves behind is read by later hooks."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
handler: Final = OpenAIChatCompletionsHandler()
request_data: Final[dict[str, object]] = {"model": "gpt-4"}
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0,
delta=Delta(content="looking", tool_calls=[
{"index": 0, "id": "call_0", "type": "function",
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
]),
finish_reason="tool_calls",
)])]
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
request_data=request_data,
stream_transform_sink=StreamTransformSink(),
)
assert "response" not in request_data
assert "responses" not in request_data
assert "guardrail_choice_index" not in request_data
@pytest.mark.asyncio
async def test_an_unfinished_parallel_call_is_kept_out_of_what_the_guardrail_reads(self) -> None:
"""A call that only announced its name must not reach the guardrail, by either route."""
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
class RecordingGuardrail(CustomGuardrail):
seen_calls: list[object] = []
seen_response: list[object] = []
async def apply_guardrail(
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
input_type: Literal["request", "response"], logging_obj: object = None,
) -> GenericGuardrailAPIInputs:
type(self).seen_calls.append(inputs.get("tool_calls"))
type(self).seen_response.append(request_data.get("response"))
return inputs
def call(index: int, arguments: str) -> dict[str, object]:
return {
"index": index, "id": f"call_{index}", "type": "function",
"function": {"name": "weather", "arguments": arguments},
}
RecordingGuardrail.seen_calls = []
RecordingGuardrail.seen_response = []
handler: Final = OpenAIChatCompletionsHandler()
request_data: Final[dict[str, object]] = {}
ended_with_one_unfinished: Final = [ModelResponseStream(choices=[StreamingChoices(
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
finish_reason="tool_calls",
)])]
await handler.process_output_streaming_response(
responses_so_far=ended_with_one_unfinished,
guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"),
litellm_logging_obj=None, request_data=request_data,
stream_transform_sink=StreamTransformSink(),
)
assert RecordingGuardrail.seen_calls == [None]
inspected: Final = RecordingGuardrail.seen_response[-1]
arguments: Final = tuple(
tool_call.function.arguments
for choice in getattr(inspected, "choices", None) or ()
for tool_call in choice.message.tool_calls or ()
)
assert arguments == ('{"city": "Paris"}',)
@pytest.mark.asyncio
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None:
handler: Final = OpenAIChatCompletionsHandler()
chunks: Final = self._two_choice_tool_call_stream_chunks()
await handler.process_output_streaming_response(
responses_so_far=chunks,
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
litellm_logging_obj=None,
deliver_ended_stream_rewrites=True,
)
arguments: Final = tuple(
"".join(call.function.arguments for chunk in chunks for choice in chunk.choices
if choice.index == index for call in choice.delta.tool_calls or ())
for index in range(2)
)
assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}')
@pytest.mark.asyncio
async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self):

View file

@ -2731,7 +2731,12 @@ class TestStreamingClientDisconnectScan:
assert [call.function.name for call in released[0].choices[0].delta.tool_calls] == ["get_weather"]
assert guardrail.received_texts == [["BLOCKME"]], guardrail.received_texts
assert guardrail.received_tool_calls == [], guardrail.received_tool_calls
assert guardrail.received_tool_calls == [[{
"id": "call_1",
"type": "function",
"index": 0,
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
}]]
@pytest.mark.asyncio
async def test_closing_after_a_released_tool_call_under_incremental_diff_scans_no_held_back_text(self):