mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 1a4f6c5542 into 60fc121ce8
This commit is contained in:
commit
d4467763b3
10 changed files with 855 additions and 134 deletions
|
|
@ -1532,6 +1532,9 @@ class CustomGuardrail(CustomLogger):
|
|||
end_time=end_time,
|
||||
duration=duration,
|
||||
masked_entity_count=masked_entity_count,
|
||||
# Set by callers that inspect one choice at a time, so the per-choice entries of an
|
||||
# n > 1 response can be told apart.
|
||||
choice_index=request_data.get("guardrail_choice_index"),
|
||||
**(tracing_detail or {}),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@ class GenAIMapper:
|
|||
LiteLLM.GUARDRAIL_CONFIDENCE_SCORE: lambda d: d.confidence_score,
|
||||
LiteLLM.GUARDRAIL_RISK_SCORE: lambda d: d.risk_score,
|
||||
LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT: lambda d: d.masked_entity_count,
|
||||
LiteLLM.GUARDRAIL_CHOICE_INDEX: lambda d: d.choice_index,
|
||||
LiteLLM.GUARDRAIL_DURATION: lambda d: d.duration,
|
||||
LiteLLM.GUARDRAIL_ID: lambda d: d.guardrail_id,
|
||||
LiteLLM.GUARDRAIL_POLICY_TEMPLATE: lambda d: d.policy_template,
|
||||
|
|
|
|||
|
|
@ -253,6 +253,7 @@ class GuardrailSpanData:
|
|||
mode: str | None = None
|
||||
status: str | None = None
|
||||
masked_entity_count: int | None = None
|
||||
choice_index: int | None = None
|
||||
provider: str | None = None
|
||||
action: str | None = None
|
||||
# The guardrail verdict / provider response (e.g. the moderation result),
|
||||
|
|
@ -313,6 +314,7 @@ class GuardrailSpanData:
|
|||
mode=_guardrail_mode_str(get("guardrail_mode")),
|
||||
status=status,
|
||||
masked_entity_count=_total_masked_entities(get("masked_entity_count")),
|
||||
choice_index=as_int(get("choice_index")),
|
||||
provider=as_str(get("guardrail_provider")),
|
||||
action=as_str(get("guardrail_action")),
|
||||
response_json=_json_or_none(response) if response is not None else None,
|
||||
|
|
|
|||
|
|
@ -317,6 +317,7 @@ class LiteLLM:
|
|||
GUARDRAIL_CONFIDENCE_SCORE: Final = "litellm.guardrail.confidence_score"
|
||||
GUARDRAIL_RISK_SCORE: Final = "litellm.guardrail.risk_score"
|
||||
GUARDRAIL_MASKED_ENTITY_COUNT: Final = "litellm.guardrail.masked_entity_count"
|
||||
GUARDRAIL_CHOICE_INDEX: Final = "litellm.guardrail.choice_index"
|
||||
GUARDRAIL_DURATION: Final = "litellm.guardrail.duration"
|
||||
GUARDRAIL_ID: Final = "litellm.guardrail.id"
|
||||
GUARDRAIL_POLICY_TEMPLATE: Final = "litellm.guardrail.policy_template"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
|
||||
|
||||
|
|
@ -34,6 +34,10 @@ class StreamTransformSink:
|
|||
mutated_text_per_choice: dict[int, str] = field(default_factory=dict)
|
||||
holdback_per_choice: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def record(self, texts: Mapping[int, str], holdback: Mapping[int, int]) -> None:
|
||||
self.mutated_text_per_choice = dict(texts)
|
||||
self.holdback_per_choice = dict(holdback)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StreamingScanKey:
|
||||
|
|
|
|||
|
|
@ -18,9 +18,11 @@ import json
|
|||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Union, cast
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
|
|
@ -50,7 +52,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
|
|||
stream_item_items,
|
||||
unappliable_request_rewrite,
|
||||
)
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
|
||||
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk, ChatCompletionToolParam
|
||||
from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
|
||||
coerce_stream_holdback_value,
|
||||
)
|
||||
|
|
@ -172,8 +174,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
|
||||
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", [])
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
|
||||
guardrailed_tool_calls: Final = guardrailed_inputs.get("tool_calls", ())
|
||||
guardrailed_tools: Final = guardrailed_inputs.get("tools")
|
||||
if guardrailed_tools is not None:
|
||||
data["tools"] = (
|
||||
|
|
@ -392,7 +394,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
request_data: dict | None = None,
|
||||
request_data: dict[str, object] | None = None,
|
||||
) -> ModelResponse:
|
||||
"""
|
||||
Process output response by applying guardrails to text content.
|
||||
|
|
@ -443,9 +445,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"response": response}
|
||||
else:
|
||||
if "response" not in request_data:
|
||||
request_data["response"] = response
|
||||
request_data.setdefault("response", response)
|
||||
|
||||
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
|
||||
|
||||
|
|
@ -465,7 +465,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
|
||||
returned_tool_calls: Final = guardrailed_inputs.get("tool_calls")
|
||||
guardrailed_tool_calls: Final[list[dict[str, object]]] = (
|
||||
cast(list[dict[str, object]], returned_tool_calls)
|
||||
|
|
@ -499,7 +499,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
user_api_key_dict: "UserAPIKeyAuth | None" = None,
|
||||
request_data: dict | None = None,
|
||||
request_data: dict[str, object] | None = None,
|
||||
stream_transform_sink: StreamTransformSink | None = None,
|
||||
deliver_ended_stream_rewrites: bool = False,
|
||||
) -> list["ModelResponseStream"]:
|
||||
|
|
@ -555,14 +555,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None",
|
||||
user_api_key_dict: "UserAPIKeyAuth | None",
|
||||
request_data: dict | None,
|
||||
request_data: dict[str, object] | None,
|
||||
deliver_ended_stream_rewrites: bool = False,
|
||||
) -> list["ModelResponseStream"]:
|
||||
"""Block-only streaming path: run the guardrail so an in-flight BLOCK can
|
||||
terminate the stream. Text rewrites are not propagated to the client here
|
||||
(see ``_process_streaming_transform`` for the incremental_diff path) unless
|
||||
``deliver_ended_stream_rewrites`` opts the ended-stream branch in."""
|
||||
has_stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
has_stream_ended: Final = deliver_ended_stream_rewrites or self._first_choice_has_finished(responses_so_far)
|
||||
|
||||
if has_stream_ended:
|
||||
await self._process_ended_stream(
|
||||
|
|
@ -609,8 +609,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"responses": responses_so_far}
|
||||
owes_responses = False # rebind-ok: the standalone dict is ours to keep
|
||||
else:
|
||||
if "responses" not in request_data:
|
||||
owes_responses = "responses" not in request_data # rebind-ok: tracks the key this scan adds
|
||||
if owes_responses:
|
||||
request_data["responses"] = responses_so_far
|
||||
|
||||
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
|
||||
|
|
@ -621,14 +623,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# Include model information from the first response if available
|
||||
if responses_so_far and hasattr(responses_so_far[0], "model") and responses_so_far[0].model:
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
try:
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
finally:
|
||||
if owes_responses:
|
||||
request_data.pop("responses", None)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", [])
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
|
||||
|
||||
# Step 4: Apply guardrailed text back to all streaming chunks
|
||||
# For each choice, replace the combined text across all chunks
|
||||
|
|
@ -661,13 +667,63 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
pre_guardrail_texts: Final = self._string_choice_contents(model_response)
|
||||
pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response)
|
||||
await self.process_output_response(
|
||||
response=model_response,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=request_data,
|
||||
inspection_responses: Final = (
|
||||
tuple(
|
||||
model_response.model_copy(update=MappingProxyType({"choices": [choice]}))
|
||||
for choice in model_response.choices
|
||||
)
|
||||
if pre_guardrail_tool_calls and len(model_response.choices) > 1
|
||||
else (model_response,)
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
inspection_request_data: Final = request_data if request_data is not None else {}
|
||||
missing: Final = object()
|
||||
prior_response: Final = inspection_request_data.get("response", missing)
|
||||
prior_responses: Final = inspection_request_data.get("responses", missing)
|
||||
prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing)
|
||||
inspection_request_data.pop("guardrail_choice_index", None)
|
||||
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
|
||||
try:
|
||||
for inspection_response in inspection_responses:
|
||||
inspection_request_data["response"] = inspection_response
|
||||
inspection_request_data["guardrail_choice_index"] = (
|
||||
inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None
|
||||
)
|
||||
inspection_request_data["responses"] = (
|
||||
[
|
||||
self._narrowed_to_choice(chunk, inspection_response.choices[0].index)
|
||||
for chunk in responses_so_far
|
||||
]
|
||||
if len(inspection_response.choices) == 1
|
||||
else responses_so_far
|
||||
)
|
||||
try:
|
||||
await self.process_output_response(
|
||||
response=inspection_response,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=inspection_request_data,
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if undeliverable is None:
|
||||
undeliverable = exc
|
||||
finally:
|
||||
if prior_response is missing:
|
||||
inspection_request_data.pop("response", None)
|
||||
else:
|
||||
inspection_request_data["response"] = prior_response
|
||||
if prior_responses is missing:
|
||||
inspection_request_data.pop("responses", None)
|
||||
else:
|
||||
inspection_request_data["responses"] = prior_responses
|
||||
if prior_choice_index is missing:
|
||||
inspection_request_data.pop("guardrail_choice_index", None)
|
||||
else:
|
||||
inspection_request_data["guardrail_choice_index"] = prior_choice_index
|
||||
if undeliverable is not None:
|
||||
raise undeliverable
|
||||
if not deliver_ended_stream_rewrites:
|
||||
return
|
||||
await self._write_ended_stream_text_rewrites(
|
||||
|
|
@ -682,6 +738,95 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _streamed_argument_fragments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> Iterator[tuple[tuple[int, int], str]]:
|
||||
for chunk in responses_so_far:
|
||||
for choice in chunk.choices:
|
||||
for call in choice.delta.tool_calls or ():
|
||||
arguments = getattr(getattr(call, "function", None), "arguments", None)
|
||||
yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else ""
|
||||
|
||||
@staticmethod
|
||||
def _parses_as_json(arguments: str) -> bool:
|
||||
if not arguments:
|
||||
return False
|
||||
try:
|
||||
json.loads(arguments)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _joined_streamed_arguments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> Mapping[tuple[int, int], str]:
|
||||
"""The argument text streamed so far per (choice index, tool call index).
|
||||
|
||||
Grouped in one pass. Rescanning every fragment once per distinct call made a sampled
|
||||
round cost the number of calls times the length of the stream, which at n=2 grows with
|
||||
the square of the streamed arguments.
|
||||
"""
|
||||
fragments: Final = sorted(
|
||||
OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far),
|
||||
key=lambda fragment: fragment[0],
|
||||
)
|
||||
return MappingProxyType(
|
||||
{call: "".join(text for _, text in group) for call, group in groupby(fragments, key=lambda f: f[0])}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool:
|
||||
"""Whether every streamed tool call now carries arguments that parse as JSON.
|
||||
|
||||
A call that has announced itself but emitted no argument text yet is still in flight, so a
|
||||
parallel call finishing first does not make the whole set complete.
|
||||
"""
|
||||
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
|
||||
if not joined:
|
||||
return False
|
||||
return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values())
|
||||
|
||||
@staticmethod
|
||||
def _streamed_call_ids(responses_so_far: Sequence["ModelResponseStream"]) -> Mapping[tuple[int, int], str]:
|
||||
"""The id each (choice index, tool call index) announced itself with."""
|
||||
choices: Final = chain.from_iterable(chunk.choices for chunk in responses_so_far)
|
||||
calls: Final = chain.from_iterable(
|
||||
((choice.index, call) for call in choice.delta.tool_calls or ()) for choice in choices
|
||||
)
|
||||
return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id})
|
||||
|
||||
@staticmethod
|
||||
def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices":
|
||||
kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished]
|
||||
message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None}))
|
||||
return choice.model_copy(update=MappingProxyType({"message": message}))
|
||||
|
||||
@staticmethod
|
||||
def _without_unfinished_tool_calls(
|
||||
assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"]
|
||||
) -> "ModelResponse":
|
||||
"""Copy of the rebuilt response without the calls still in flight when the stream ended.
|
||||
|
||||
Rebuilding gives a call that never streamed complete arguments an empty JSON object, which
|
||||
reads to a guardrail as a real call taking no arguments rather than an unfinished one. The
|
||||
calls are matched by the id they announced, since the rebuilt order need not follow the
|
||||
order the stream gave them.
|
||||
"""
|
||||
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
|
||||
unfinished: Final = frozenset(
|
||||
call_id
|
||||
for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items()
|
||||
if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, ""))
|
||||
)
|
||||
if not unfinished:
|
||||
return assembled
|
||||
pruned: Final = [
|
||||
OpenAIChatCompletionsHandler._choice_without(choice, unfinished) for choice in assembled.choices
|
||||
]
|
||||
return assembled.model_copy(update=MappingProxyType({"choices": pruned}))
|
||||
|
||||
@staticmethod
|
||||
def _rebuild_ended_stream_per_choice(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
|
|
@ -706,6 +851,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
)
|
||||
for index in choice_indices
|
||||
)
|
||||
if not rebuilt_by_index:
|
||||
whole: Final = stream_chunk_builder(chunks=list(responses_so_far), logging_obj=litellm_logging_obj)
|
||||
return cast(ModelResponse, whole) if whole is not None else ModelResponse(choices=[])
|
||||
(_, base_response), *_ = rebuilt_by_index
|
||||
stitched_choices: Final = [
|
||||
rebuilt.choices[0].model_copy(update=MappingProxyType({"index": index}))
|
||||
|
|
@ -763,7 +911,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_to_apply: "CustomGuardrail",
|
||||
litellm_logging_obj: "LiteLLMLoggingObj | None",
|
||||
user_api_key_dict: "UserAPIKeyAuth | None",
|
||||
request_data: dict | None,
|
||||
request_data: dict[str, object] | None,
|
||||
sink: StreamTransformSink,
|
||||
) -> None:
|
||||
"""Run the guardrail over the raw accumulated text and report the
|
||||
|
|
@ -775,8 +923,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
"""
|
||||
raw_by_index: Final = self._accumulate_string_content_by_choice_index(responses_so_far)
|
||||
if not raw_by_index:
|
||||
sink.mutated_text_per_choice = {}
|
||||
sink.holdback_per_choice = {}
|
||||
sink.record(MappingProxyType({}), MappingProxyType({}))
|
||||
return
|
||||
|
||||
# Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits
|
||||
|
|
@ -788,44 +935,150 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
indices: Final = sorted(raw_by_index.keys())
|
||||
texts_to_check: Final = [raw_by_index[i] for i in indices]
|
||||
|
||||
if request_data is None:
|
||||
request_data = {"responses": responses_so_far}
|
||||
elif "responses" not in request_data:
|
||||
request_data["responses"] = responses_so_far
|
||||
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
|
||||
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if responses_so_far and getattr(responses_so_far[0], "model", None):
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
resolved_request_data: Final[dict[str, object]] = (
|
||||
request_data if request_data is not None else {"responses": responses_so_far}
|
||||
)
|
||||
missing: Final = object()
|
||||
prior_response: Final = resolved_request_data.get("response", missing)
|
||||
prior_responses: Final = resolved_request_data.get("responses", missing)
|
||||
prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing)
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if len(indices) == 1:
|
||||
resolved_request_data["guardrail_choice_index"] = indices[0]
|
||||
if "responses" not in resolved_request_data:
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict)
|
||||
try:
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if self._streamed_tool_call_fingerprints(responses_so_far):
|
||||
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
|
||||
assembled: Final = (
|
||||
None
|
||||
if not stream_ended and not tool_arguments_complete
|
||||
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
)
|
||||
inspected: Final = (
|
||||
None
|
||||
if assembled is None
|
||||
else assembled
|
||||
if tool_arguments_complete
|
||||
else self._without_unfinished_tool_calls(assembled, responses_so_far)
|
||||
)
|
||||
if inspected is not None:
|
||||
resolved_request_data["response"] = inspected
|
||||
if inspected is not None and len(inspected.choices) > 1:
|
||||
choice_rounds: Final = tuple(
|
||||
(
|
||||
choice,
|
||||
StreamTransformSink(),
|
||||
[self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far],
|
||||
)
|
||||
for choice in inspected.choices
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import (
|
||||
UndeliverableStreamRewrite,
|
||||
)
|
||||
|
||||
returned_texts = guardrailed_inputs.get("texts")
|
||||
# No "texts" key means the guardrail made no change (action NONE): the raw
|
||||
# accumulated text is the guardrailed text. A present-but-shorter list is a
|
||||
# guardrail contract violation; those choices are omitted below (withheld,
|
||||
# not emitted raw) so a malformed response fails closed instead of leaking.
|
||||
if returned_texts is None:
|
||||
returned_texts = texts_to_check
|
||||
elif len(returned_texts) < len(texts_to_check):
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
|
||||
"streaming transform path; withholding the unmatched choices to fail closed.",
|
||||
len(returned_texts),
|
||||
len(texts_to_check),
|
||||
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: first failure
|
||||
try:
|
||||
for choice, choice_sink, choice_chunks in choice_rounds:
|
||||
resolved_request_data["response"] = inspected.model_copy(
|
||||
update=MappingProxyType({"choices": [choice]})
|
||||
)
|
||||
resolved_request_data["responses"] = choice_chunks
|
||||
resolved_request_data["guardrail_choice_index"] = choice.index
|
||||
try:
|
||||
await self._process_streaming_transform(
|
||||
responses_so_far=choice_chunks,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=resolved_request_data,
|
||||
sink=choice_sink,
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if transform_undeliverable is None:
|
||||
transform_undeliverable = exc
|
||||
finally:
|
||||
resolved_request_data["response"] = assembled
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if transform_undeliverable is not None:
|
||||
raise transform_undeliverable
|
||||
merged_texts: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
)
|
||||
)
|
||||
)
|
||||
merged_holdback: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
)
|
||||
)
|
||||
)
|
||||
sink.record(merged_texts, merged_holdback)
|
||||
return
|
||||
if assembled is not None and tool_arguments_complete:
|
||||
tool_calls: Final = chain.from_iterable(
|
||||
choice.message.tool_calls or () for choice in assembled.choices
|
||||
)
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
MappingProxyType({"index": index, **converted})
|
||||
for index, tool_call in enumerate(tool_calls)
|
||||
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
|
||||
)
|
||||
)
|
||||
if responses_so_far and getattr(responses_so_far[0], "model", None):
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=resolved_request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or []
|
||||
sink.mutated_text_per_choice = {
|
||||
idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)
|
||||
}
|
||||
sink.holdback_per_choice = {
|
||||
indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)
|
||||
}
|
||||
returned_texts = guardrailed_inputs.get("texts")
|
||||
# No "texts" key means the guardrail made no change (action NONE): the raw
|
||||
# accumulated text is the guardrailed text. A present-but-shorter list is a
|
||||
# guardrail contract violation; those choices are omitted below (withheld,
|
||||
# not emitted raw) so a malformed response fails closed instead of leaking.
|
||||
if returned_texts is None:
|
||||
returned_texts = texts_to_check
|
||||
elif len(returned_texts) < len(texts_to_check):
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
|
||||
"streaming transform path; withholding the unmatched choices to fail closed.",
|
||||
len(returned_texts),
|
||||
len(texts_to_check),
|
||||
)
|
||||
|
||||
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or ()
|
||||
transformed_texts: Final = MappingProxyType(
|
||||
{idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)}
|
||||
)
|
||||
transformed_holdback: Final = MappingProxyType(
|
||||
{
|
||||
indices[i]: coerce_stream_holdback_value(holdback[i])
|
||||
for i in range(len(indices))
|
||||
if i < len(holdback)
|
||||
}
|
||||
)
|
||||
sink.record(transformed_texts, transformed_holdback)
|
||||
finally:
|
||||
for key, prior in (
|
||||
("response", prior_response),
|
||||
("responses", prior_responses),
|
||||
("guardrail_choice_index", prior_choice_index),
|
||||
):
|
||||
if prior is missing:
|
||||
resolved_request_data.pop(key, None)
|
||||
else:
|
||||
resolved_request_data[key] = prior
|
||||
|
||||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream))
|
||||
|
|
@ -1142,20 +1395,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
def _function_tool_call_fragments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]:
|
||||
"""Group the stream's function tool-call fragments by their tool-call index, in
|
||||
the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping
|
||||
only the indices the builder keeps (an id and a name somewhere in the stream)."""
|
||||
fragments: Final = tuple(
|
||||
tool_call
|
||||
(choice.index, tool_call)
|
||||
for response in responses_so_far
|
||||
for choice in response.choices
|
||||
for tool_call in choice.delta.tool_calls or ()
|
||||
if isinstance(tool_call, ChatCompletionDeltaToolCall)
|
||||
)
|
||||
identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id)
|
||||
named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name)
|
||||
identified: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.id)
|
||||
named: Final = frozenset((choice, fragment.index) for choice, fragment in fragments if fragment.function.name)
|
||||
return tuple(
|
||||
tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named)
|
||||
tuple(fragment for choice, fragment in fragments if (choice, fragment.index) == key)
|
||||
for key in sorted(identified & named)
|
||||
)
|
||||
|
||||
def _write_ended_stream_tool_call_rewrites(
|
||||
|
|
@ -1165,28 +1416,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
pre_guardrail_tool_calls: tuple[tuple[str | None, str], ...],
|
||||
guardrail_name: str,
|
||||
) -> None:
|
||||
"""Write ended-stream guardrail tool-call rewrites back across the buffered
|
||||
chunks: the rewritten name and full arguments land in the tool call's first
|
||||
fragment and the arguments of its later fragments are blanked, mirroring the
|
||||
text write-back. A rewrite on a stream carrying more than one distinct choice
|
||||
index, or whose fragments do not line up with the rebuilt tool calls, is
|
||||
reported as undeliverable, so the pipeline executor discards it and releases
|
||||
the original chunks."""
|
||||
post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response)
|
||||
if post_guardrail_tool_calls == pre_guardrail_tool_calls:
|
||||
return
|
||||
stream_choice_indices: Final = frozenset(
|
||||
choice.index for response in responses_so_far for choice in response.choices
|
||||
)
|
||||
fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far)
|
||||
if len(stream_choice_indices) != 1:
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
raise UndeliverableStreamRewrite(
|
||||
guardrail_name,
|
||||
f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written "
|
||||
"back on single-choice streams",
|
||||
)
|
||||
if len(fragments_by_tool_call) != len(post_guardrail_tool_calls):
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
|
|
@ -1209,7 +1442,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
async def _apply_guardrail_responses_to_output_streaming(
|
||||
self,
|
||||
responses: list["ModelResponseStream"],
|
||||
guardrailed_texts: list[str],
|
||||
guardrailed_texts: Sequence[str],
|
||||
task_mappings: list[tuple[int, int | None]],
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -1227,12 +1460,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
Override this method to customize how responses are applied to streaming responses.
|
||||
"""
|
||||
# Build a mapping of what guardrailed text to use for each (choice_idx, content_idx)
|
||||
guardrail_map: Final[dict[tuple[int, int | None], str]] = {}
|
||||
for task_idx, guardrail_response in enumerate(guardrailed_texts):
|
||||
mapping = task_mappings[task_idx]
|
||||
choice_idx = cast(int, mapping[0])
|
||||
content_idx_optional = cast(int | None, mapping[1])
|
||||
guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response
|
||||
guardrail_map: Final = MappingProxyType(
|
||||
{task_mappings[index]: text for index, text in enumerate(guardrailed_texts)}
|
||||
)
|
||||
|
||||
# Track which choices we've already set the guardrailed text for
|
||||
# Key: (choice_idx, content_idx), Value: boolean (True if already set)
|
||||
|
|
@ -1242,34 +1472,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# choice by its index field: on n>1 streams a chunk usually carries one
|
||||
# choice at list position 0 whose index names the logical choice.
|
||||
for response in responses:
|
||||
for choice in response.choices:
|
||||
if isinstance(choice, litellm.StreamingChoices):
|
||||
content = choice.delta.content
|
||||
elif isinstance(choice, litellm.Choices):
|
||||
content = choice.message.content
|
||||
else:
|
||||
continue
|
||||
|
||||
for choice, message in (
|
||||
(choice, choice.delta if isinstance(choice, StreamingChoices) else choice.message)
|
||||
for choice in response.choices
|
||||
if isinstance(choice, (StreamingChoices, Choices))
|
||||
):
|
||||
content = message.content
|
||||
if content is None:
|
||||
continue
|
||||
|
||||
if isinstance(content, str):
|
||||
# String content
|
||||
str_key: tuple[int, int | None] = (choice.index, None)
|
||||
if str_key in guardrail_map:
|
||||
if str_key not in already_set:
|
||||
# First chunk - set the complete guardrailed text
|
||||
if isinstance(choice, litellm.StreamingChoices):
|
||||
choice.delta.content = guardrail_map[str_key]
|
||||
elif isinstance(choice, litellm.Choices):
|
||||
choice.message.content = guardrail_map[str_key]
|
||||
already_set[str_key] = True
|
||||
else:
|
||||
# Subsequent chunks - clear the content
|
||||
if isinstance(choice, litellm.StreamingChoices):
|
||||
choice.delta.content = ""
|
||||
elif isinstance(choice, litellm.Choices):
|
||||
choice.message.content = ""
|
||||
message.content = guardrail_map[str_key] if str_key not in already_set else ""
|
||||
already_set[str_key] = True
|
||||
|
||||
elif isinstance(content, list):
|
||||
# List content - handle each content item
|
||||
|
|
|
|||
|
|
@ -3591,6 +3591,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"end_time",
|
||||
"duration",
|
||||
"masked_entity_count",
|
||||
"choice_index",
|
||||
"guardrail_id",
|
||||
"policy_template",
|
||||
"detection_method",
|
||||
|
|
@ -3631,6 +3632,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
|
|||
}
|
||||
"""
|
||||
|
||||
choice_index: ReadOnly[int | None]
|
||||
"""Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice."""
|
||||
|
||||
guardrail_id: str | None
|
||||
"""Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'"""
|
||||
|
||||
|
|
|
|||
|
|
@ -344,6 +344,19 @@ def test_guardrail_block_span_is_error_and_carries_verdict():
|
|||
assert a[LiteLLM.GUARDRAIL_MASKED_ENTITY_COUNT] == 2
|
||||
|
||||
|
||||
def test_guardrail_span_carries_the_choice_index():
|
||||
"""A per-choice guardrail entry must say on the span which choice it inspected."""
|
||||
engine, exporter = _engine()
|
||||
engine.emit(
|
||||
SpanRole.GUARDRAIL,
|
||||
GuardrailSpanData.from_logging_entry(
|
||||
{"guardrail_name": "g", "guardrail_status": "success", "choice_index": 1}
|
||||
),
|
||||
)
|
||||
(span,) = exporter.get_finished_spans()
|
||||
assert span.attributes[LiteLLM.GUARDRAIL_CHOICE_INDEX] == 1
|
||||
|
||||
|
||||
def test_guardrail_success_span_is_unset():
|
||||
"""On success the status is left UNSET (semconv default) — not forced OK."""
|
||||
engine, exporter = _engine()
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ with guardrail transformations, including tool calls.
|
|||
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Literal, Optional
|
||||
from typing import Any, Final, Literal, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -1395,29 +1395,501 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
|
||||
assert [
|
||||
(tool_call["id"], tool_call["function"]["arguments"])
|
||||
for tool_call in guardrail.seen_inputs[-1]["tool_calls"]
|
||||
] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')]
|
||||
[(tool_call["id"], tool_call["function"]["arguments"]) for tool_call in inputs["tool_calls"]]
|
||||
for inputs in guardrail.seen_inputs
|
||||
] == [[("call_1", '{"fruit": "persimmon"}')], [("call_2", '{"fruit": "durian"}')]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self):
|
||||
@pytest.mark.parametrize("transform", [False, True])
|
||||
async def test_each_choice_refreshes_shared_guardrail_response_context(self, transform: bool) -> None:
|
||||
from fastapi import HTTPException
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class ContextGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
response: Final = request_data["response"]
|
||||
assert isinstance(response, ModelResponse)
|
||||
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), response.choices[0].index)
|
||||
if response.choices[0].message.content == "forbidden":
|
||||
raise HTTPException(status_code=400, detail="later choice blocked")
|
||||
return inputs
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("allowed", "forbidden"))])]
|
||||
request_data: Final = {"metadata": {"trace": "retained"}}
|
||||
with pytest.raises(HTTPException, match="later choice blocked"):
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=ContextGuardrail(guardrail_name="context"),
|
||||
request_data=request_data, deliver_ended_stream_rewrites=True,
|
||||
stream_transform_sink=StreamTransformSink() if transform else None,
|
||||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
assert request_data["metadata"]["trace"] == "retained"
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
|
||||
"""An undeliverable rewrite on one choice must not leave later choices uninspected.
|
||||
|
||||
The caller releases the original chunks when it catches UndeliverableStreamRewrite, so
|
||||
bailing out of the loop would let a request with n > 1 pair a redactable choice with a
|
||||
prohibited one and have the whole response delivered unchecked.
|
||||
"""
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
handler = OpenAIChatCompletionsHandler()
|
||||
chunks = self._two_choice_tool_call_stream_chunks()
|
||||
class UndeliverableOnFirstChoice(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
response: Final = request_data["response"]
|
||||
assert isinstance(response, ModelResponse)
|
||||
index: Final = response.choices[0].index
|
||||
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
|
||||
if index == 0:
|
||||
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
|
||||
return inputs
|
||||
|
||||
with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised:
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("redacted", "prohibited"))])]
|
||||
request_data: Final = {}
|
||||
with pytest.raises(UndeliverableStreamRewrite):
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
|
||||
request_data=request_data, deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None:
|
||||
"""The transform path runs its own per-choice loop and needs the same rule."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class UndeliverableOnFirstChoice(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
response: Final = request_data["response"]
|
||||
assert isinstance(response, ModelResponse)
|
||||
index: Final = response.choices[0].index
|
||||
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
|
||||
if index == 0:
|
||||
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
|
||||
return inputs
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("redacted", "prohibited"))])]
|
||||
request_data: Final = {}
|
||||
with pytest.raises(UndeliverableStreamRewrite):
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
|
||||
request_data=request_data, deliver_ended_stream_rewrites=True,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None:
|
||||
"""A buffered stream has no finish_reason yet, so the ended path must key off the opt-in.
|
||||
|
||||
Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered
|
||||
caller on the in-flight path, where rewrites are never written back.
|
||||
"""
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class PathProbe(CustomGuardrail):
|
||||
saw_assembled_response: bool = False
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse)
|
||||
return inputs
|
||||
|
||||
chunks: Final = [
|
||||
ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)]
|
||||
)
|
||||
]
|
||||
assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=PathProbe(guardrail_name="probe"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert PathProbe.saw_assembled_response is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None:
|
||||
"""The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks.
|
||||
|
||||
Reporting an empty holdback here would release text the guardrail asked to withhold.
|
||||
"""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class HoldingGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
texts: Final = inputs.get("texts") or ()
|
||||
return {**inputs, "stream_holdback_chars": [len(text) for text in texts]}
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("alpha", "bravo-long"))])]
|
||||
sink: Final = StreamTransformSink()
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"),
|
||||
litellm_logging_obj=None,
|
||||
stream_transform_sink=sink,
|
||||
)
|
||||
assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None:
|
||||
"""Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent.
|
||||
|
||||
Sending them would hand the guardrail something like `{"ci`, which main never did.
|
||||
"""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(inputs.get("tool_calls"))
|
||||
return inputs
|
||||
|
||||
def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream:
|
||||
return ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments},
|
||||
}]), finish_reason=finish_reason,
|
||||
)])
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
|
||||
partial: Final = [chunk("looking", '{"ci', None)]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=partial, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert RecordingGuardrail.seen == [None]
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=complete, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
sent: Final = RecordingGuardrail.seen[-1]
|
||||
assert sent is not None
|
||||
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None:
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(inputs.get("tool_calls"))
|
||||
return inputs
|
||||
|
||||
def call(index: int, arguments: str) -> dict[str, object]:
|
||||
return {
|
||||
"index": index, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments},
|
||||
}
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
|
||||
one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
|
||||
finish_reason=None,
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=one_still_empty, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert RecordingGuardrail.seen == [None]
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
both_complete: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=both_complete, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
sent: Final = RecordingGuardrail.seen[-1]
|
||||
assert sent is not None
|
||||
assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_buffered_stream_that_carried_no_choices_is_not_an_error(self) -> None:
|
||||
"""A stream can end carrying only usage, and a policy step still runs over it."""
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[], usage={"prompt_tokens": 3, "completion_tokens": 0,
|
||||
"total_tokens": 3})]
|
||||
result: Final = await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
request_data={},
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert result is chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_round_with_nothing_to_judge_costs_one_call_at_n2(self) -> None:
|
||||
"""Per-choice rounds buy tool-call context. While no call is complete there is none to buy,
|
||||
and charging a vendor once per choice for every sampled round buys nothing."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class CountingGuardrail(CustomGuardrail):
|
||||
calls: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).calls.append(inputs.get("texts"))
|
||||
return inputs
|
||||
|
||||
def choice(index: int, arguments: str) -> StreamingChoices:
|
||||
return StreamingChoices(
|
||||
index=index,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments}},
|
||||
]),
|
||||
finish_reason=None,
|
||||
)
|
||||
|
||||
assert raised.value.guardrail_name == "test"
|
||||
assert raised.value.reason == (
|
||||
"the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams"
|
||||
CountingGuardrail.calls = []
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=[ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"ci')])],
|
||||
guardrail_to_apply=CountingGuardrail(guardrail_name="counting"),
|
||||
litellm_logging_obj=None, request_data={},
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert len(CountingGuardrail.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None:
|
||||
"""Half a JSON object is not a call a guardrail can judge, at any number of choices."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(request_data.get("response"))
|
||||
return inputs
|
||||
|
||||
def choice(index: int, arguments: str, label: str) -> StreamingChoices:
|
||||
return StreamingChoices(
|
||||
index=index,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_" + label, "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=[ModelResponseStream(choices=[
|
||||
choice(0, '{"city": "Pa', "partial"), choice(1, '{"city": "Rome"}', "rome"),
|
||||
])],
|
||||
guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"),
|
||||
litellm_logging_obj=None, request_data={},
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
arguments: Final = tuple(
|
||||
tool_call.function.arguments
|
||||
for response in RecordingGuardrail.seen
|
||||
for single in getattr(response, "choices", None) or ()
|
||||
for tool_call in single.message.tool_calls or ()
|
||||
)
|
||||
assert arguments == ('{"city": "Rome"}',)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_caller_cannot_set_the_choice_index_label(self) -> None:
|
||||
"""The label says which choice the round inspected, so only the round may set it."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class LabelReadingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(request_data.get("guardrail_choice_index"))
|
||||
return inputs
|
||||
|
||||
LabelReadingGuardrail.seen = []
|
||||
request_data: Final[dict[str, object]] = {"model": "gpt-4", "guardrail_choice_index": 7}
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=LabelReadingGuardrail(guardrail_name="label"),
|
||||
litellm_logging_obj=None,
|
||||
request_data=request_data,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert LabelReadingGuardrail.seen == [0]
|
||||
assert request_data["guardrail_choice_index"] == 7
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_transform_round_leaves_the_callers_request_data_as_it_found_it(self) -> None:
|
||||
"""The round owns these keys only while it runs; what it leaves behind is read by later hooks."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
request_data: Final[dict[str, object]] = {"model": "gpt-4"}
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
request_data=request_data,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
assert "guardrail_choice_index" not in request_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unfinished_parallel_call_is_kept_out_of_what_the_guardrail_reads(self) -> None:
|
||||
"""A call that only announced its name must not reach the guardrail, by either route."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen_calls: list[object] = []
|
||||
seen_response: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen_calls.append(inputs.get("tool_calls"))
|
||||
type(self).seen_response.append(request_data.get("response"))
|
||||
return inputs
|
||||
|
||||
def call(index: int, arguments: str) -> dict[str, object]:
|
||||
return {
|
||||
"index": index, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments},
|
||||
}
|
||||
|
||||
RecordingGuardrail.seen_calls = []
|
||||
RecordingGuardrail.seen_response = []
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
request_data: Final[dict[str, object]] = {}
|
||||
ended_with_one_unfinished: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=ended_with_one_unfinished,
|
||||
guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"),
|
||||
litellm_logging_obj=None, request_data=request_data,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert RecordingGuardrail.seen_calls == [None]
|
||||
inspected: Final = RecordingGuardrail.seen_response[-1]
|
||||
arguments: Final = tuple(
|
||||
tool_call.function.arguments
|
||||
for choice in getattr(inspected, "choices", None) or ()
|
||||
for tool_call in choice.message.tool_calls or ()
|
||||
)
|
||||
assert arguments == ('{"city": "Paris"}',)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None:
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
chunks: Final = self._two_choice_tool_call_stream_chunks()
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
arguments: Final = tuple(
|
||||
"".join(call.function.arguments for chunk in chunks for choice in chunk.choices
|
||||
if choice.index == index for call in choice.delta.tool_calls or ())
|
||||
for index in range(2)
|
||||
)
|
||||
assert arguments == ('{"fruit": "PERSIMMON"}', '{"fruit": "DURIAN"}')
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self):
|
||||
|
|
|
|||
|
|
@ -2731,7 +2731,12 @@ class TestStreamingClientDisconnectScan:
|
|||
|
||||
assert [call.function.name for call in released[0].choices[0].delta.tool_calls] == ["get_weather"]
|
||||
assert guardrail.received_texts == [["BLOCKME"]], guardrail.received_texts
|
||||
assert guardrail.received_tool_calls == [], guardrail.received_tool_calls
|
||||
assert guardrail.received_tool_calls == [[{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"index": 0,
|
||||
"function": {"name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
}]]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_closing_after_a_released_tool_call_under_incremental_diff_scans_no_held_back_text(self):
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue