mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
Merge remote-tracking branch 'origin/litellm_guardrail_choice_isolation' into litellm_guardrail_stream_inspection
This commit is contained in:
commit
f17c443548
2 changed files with 345 additions and 143 deletions
|
|
@ -14,12 +14,11 @@ Pattern Overview:
|
|||
This pattern can be replicated for other message formats (e.g., Anthropic).
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from itertools import chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Union, cast
|
||||
|
||||
|
|
@ -610,8 +609,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# create a standalone dict (SDK / direct-call path).
|
||||
if request_data is None:
|
||||
request_data = {"responses": responses_so_far}
|
||||
owes_responses = False # rebind-ok: the standalone dict is ours to keep
|
||||
else:
|
||||
if "responses" not in request_data:
|
||||
owes_responses = "responses" not in request_data # rebind-ok: tracks the key this scan adds
|
||||
if owes_responses:
|
||||
request_data["responses"] = responses_so_far
|
||||
|
||||
self.merge_user_api_key_metadata_into_request(request_data, user_api_key_dict)
|
||||
|
|
@ -622,12 +623,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
# Include model information from the first response if available
|
||||
if responses_so_far and hasattr(responses_so_far[0], "model") and responses_so_far[0].model:
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
try:
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
finally:
|
||||
if owes_responses:
|
||||
request_data.pop("responses", None)
|
||||
|
||||
guardrailed_texts: Final = guardrailed_inputs.get("texts", ())
|
||||
|
||||
|
|
@ -677,6 +682,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
prior_response: Final = inspection_request_data.get("response", missing)
|
||||
prior_responses: Final = inspection_request_data.get("responses", missing)
|
||||
prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing)
|
||||
inspection_request_data.pop("guardrail_choice_index", None)
|
||||
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
|
||||
try:
|
||||
for inspection_response in inspection_responses:
|
||||
|
|
@ -732,12 +738,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]:
|
||||
for chunk in responses_so_far:
|
||||
for choice in chunk.choices:
|
||||
yield choice.index
|
||||
|
||||
@staticmethod
|
||||
def _streamed_argument_fragments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
|
|
@ -762,13 +762,18 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
def _joined_streamed_arguments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> Mapping[tuple[int, int], str]:
|
||||
"""The argument text streamed so far per (choice index, tool call index)."""
|
||||
fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far))
|
||||
"""The argument text streamed so far per (choice index, tool call index).
|
||||
|
||||
Grouped in one pass. Rescanning every fragment once per distinct call made a sampled
|
||||
round cost the number of calls times the length of the stream, which at n=2 grows with
|
||||
the square of the streamed arguments.
|
||||
"""
|
||||
fragments: Final = sorted(
|
||||
OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far),
|
||||
key=lambda fragment: fragment[0],
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
call: "".join(text for other, text in fragments if other == call)
|
||||
for call in frozenset(call for call, _ in fragments)
|
||||
}
|
||||
{call: "".join(text for _, text in group) for call, group in groupby(fragments, key=lambda f: f[0])}
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -783,27 +788,44 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
return False
|
||||
return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values())
|
||||
|
||||
@staticmethod
|
||||
def _streamed_call_ids(responses_so_far: Sequence["ModelResponseStream"]) -> Mapping[tuple[int, int], str]:
|
||||
"""The id each (choice index, tool call index) announced itself with."""
|
||||
choices: Final = chain.from_iterable(chunk.choices for chunk in responses_so_far)
|
||||
calls: Final = chain.from_iterable(
|
||||
((choice.index, call) for call in choice.delta.tool_calls or ()) for choice in choices
|
||||
)
|
||||
return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id})
|
||||
|
||||
@staticmethod
|
||||
def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices":
|
||||
kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished]
|
||||
message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None}))
|
||||
return choice.model_copy(update=MappingProxyType({"message": message}))
|
||||
|
||||
@staticmethod
|
||||
def _without_unfinished_tool_calls(
|
||||
assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"]
|
||||
) -> "ModelResponse":
|
||||
"""Copy of the rebuilt response without the calls still in flight when the stream ended.
|
||||
|
||||
Rebuilding gives a call that never streamed any argument text an empty JSON object, which
|
||||
reads to a guardrail as a real call taking no arguments rather than an unfinished one.
|
||||
Rebuilding gives a call that never streamed complete arguments an empty JSON object, which
|
||||
reads to a guardrail as a real call taking no arguments rather than an unfinished one. The
|
||||
calls are matched by the id they announced, since the rebuilt order need not follow the
|
||||
order the stream gave them.
|
||||
"""
|
||||
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
|
||||
pruned: Final = copy.deepcopy(assembled)
|
||||
for choice in pruned.choices:
|
||||
streamed = choice.message.tool_calls or ()
|
||||
kept = [
|
||||
call
|
||||
for position, call in enumerate(streamed)
|
||||
if OpenAIChatCompletionsHandler._parses_as_json(joined.get((choice.index, position), ""))
|
||||
]
|
||||
if len(kept) != len(streamed):
|
||||
choice.message.tool_calls = kept or None
|
||||
return pruned
|
||||
unfinished: Final = frozenset(
|
||||
call_id
|
||||
for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items()
|
||||
if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, ""))
|
||||
)
|
||||
if not unfinished:
|
||||
return assembled
|
||||
pruned: Final = [
|
||||
OpenAIChatCompletionsHandler._choice_without(choice, unfinished) for choice in assembled.choices
|
||||
]
|
||||
return assembled.model_copy(update=MappingProxyType({"choices": pruned}))
|
||||
|
||||
@staticmethod
|
||||
def _rebuild_ended_stream_per_choice(
|
||||
|
|
@ -829,6 +851,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
)
|
||||
for index in choice_indices
|
||||
)
|
||||
if not rebuilt_by_index:
|
||||
whole: Final = stream_chunk_builder(chunks=list(responses_so_far), logging_obj=litellm_logging_obj)
|
||||
return cast(ModelResponse, whole) if whole is not None else ModelResponse(choices=[])
|
||||
(_, base_response), *_ = rebuilt_by_index
|
||||
stitched_choices: Final = [
|
||||
rebuilt.choices[0].model_copy(update=MappingProxyType({"index": index}))
|
||||
|
|
@ -913,122 +938,147 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
resolved_request_data: Final[dict[str, object]] = (
|
||||
request_data if request_data is not None else {"responses": responses_so_far}
|
||||
)
|
||||
missing: Final = object()
|
||||
prior_response: Final = resolved_request_data.get("response", missing)
|
||||
prior_responses: Final = resolved_request_data.get("responses", missing)
|
||||
prior_choice_index: Final = resolved_request_data.get("guardrail_choice_index", missing)
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if len(indices) == 1:
|
||||
resolved_request_data["guardrail_choice_index"] = indices[0]
|
||||
if "responses" not in resolved_request_data:
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
self.merge_user_api_key_metadata_into_request(resolved_request_data, user_api_key_dict)
|
||||
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if self._streamed_tool_call_fingerprints(responses_so_far):
|
||||
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far))
|
||||
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
|
||||
assembled: Final = (
|
||||
None
|
||||
if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete
|
||||
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
)
|
||||
if assembled is not None:
|
||||
resolved_request_data["response"] = (
|
||||
assembled
|
||||
try:
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if self._streamed_tool_call_fingerprints(responses_so_far):
|
||||
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
|
||||
assembled: Final = (
|
||||
None
|
||||
if not stream_ended and not tool_arguments_complete
|
||||
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
)
|
||||
inspected: Final = (
|
||||
None
|
||||
if assembled is None
|
||||
else assembled
|
||||
if tool_arguments_complete
|
||||
else self._without_unfinished_tool_calls(assembled, responses_so_far)
|
||||
)
|
||||
if assembled is not None and len(assembled.choices) > 1:
|
||||
choice_rounds: Final = tuple(
|
||||
(
|
||||
choice,
|
||||
StreamTransformSink(),
|
||||
[self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far],
|
||||
)
|
||||
for choice in assembled.choices
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import (
|
||||
UndeliverableStreamRewrite,
|
||||
)
|
||||
|
||||
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
|
||||
try:
|
||||
for choice, choice_sink, choice_chunks in choice_rounds:
|
||||
resolved_request_data["response"] = assembled.model_copy(
|
||||
update=MappingProxyType({"choices": [choice]})
|
||||
if inspected is not None:
|
||||
resolved_request_data["response"] = inspected
|
||||
if inspected is not None and len(inspected.choices) > 1:
|
||||
choice_rounds: Final = tuple(
|
||||
(
|
||||
choice,
|
||||
StreamTransformSink(),
|
||||
[self._narrowed_to_choice(chunk, choice.index) for chunk in responses_so_far],
|
||||
)
|
||||
resolved_request_data["responses"] = choice_chunks
|
||||
resolved_request_data["guardrail_choice_index"] = choice.index
|
||||
try:
|
||||
await self._process_streaming_transform(
|
||||
responses_so_far=choice_chunks,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=resolved_request_data,
|
||||
sink=choice_sink,
|
||||
for choice in inspected.choices
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import (
|
||||
UndeliverableStreamRewrite,
|
||||
)
|
||||
|
||||
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: first failure
|
||||
try:
|
||||
for choice, choice_sink, choice_chunks in choice_rounds:
|
||||
resolved_request_data["response"] = inspected.model_copy(
|
||||
update=MappingProxyType({"choices": [choice]})
|
||||
)
|
||||
resolved_request_data["responses"] = choice_chunks
|
||||
resolved_request_data["guardrail_choice_index"] = choice.index
|
||||
try:
|
||||
await self._process_streaming_transform(
|
||||
responses_so_far=choice_chunks,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=resolved_request_data,
|
||||
sink=choice_sink,
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if transform_undeliverable is None:
|
||||
transform_undeliverable = exc
|
||||
finally:
|
||||
resolved_request_data["response"] = assembled
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if transform_undeliverable is not None:
|
||||
raise transform_undeliverable
|
||||
merged_texts: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if transform_undeliverable is None:
|
||||
transform_undeliverable = exc
|
||||
finally:
|
||||
resolved_request_data["response"] = assembled
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if transform_undeliverable is not None:
|
||||
raise transform_undeliverable
|
||||
merged_texts: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.mutated_text_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
)
|
||||
)
|
||||
)
|
||||
merged_holdback: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
merged_holdback: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
choice_sink.holdback_per_choice.items() for _, choice_sink, _ in choice_rounds
|
||||
)
|
||||
)
|
||||
)
|
||||
)
|
||||
sink.record(merged_texts, merged_holdback)
|
||||
return
|
||||
if assembled is not None and tool_arguments_complete:
|
||||
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
MappingProxyType({"index": index, **converted})
|
||||
for index, tool_call in enumerate(tool_calls)
|
||||
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
|
||||
sink.record(merged_texts, merged_holdback)
|
||||
return
|
||||
if assembled is not None and tool_arguments_complete:
|
||||
tool_calls: Final = chain.from_iterable(
|
||||
choice.message.tool_calls or () for choice in assembled.choices
|
||||
)
|
||||
)
|
||||
if responses_so_far and getattr(responses_so_far[0], "model", None):
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=resolved_request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
returned_texts = guardrailed_inputs.get("texts")
|
||||
# No "texts" key means the guardrail made no change (action NONE): the raw
|
||||
# accumulated text is the guardrailed text. A present-but-shorter list is a
|
||||
# guardrail contract violation; those choices are omitted below (withheld,
|
||||
# not emitted raw) so a malformed response fails closed instead of leaking.
|
||||
if returned_texts is None:
|
||||
returned_texts = texts_to_check
|
||||
elif len(returned_texts) < len(texts_to_check):
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
|
||||
"streaming transform path; withholding the unmatched choices to fail closed.",
|
||||
len(returned_texts),
|
||||
len(texts_to_check),
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
MappingProxyType({"index": index, **converted})
|
||||
for index, tool_call in enumerate(tool_calls)
|
||||
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
|
||||
)
|
||||
)
|
||||
if responses_so_far and getattr(responses_so_far[0], "model", None):
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=resolved_request_data,
|
||||
input_type="response",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
|
||||
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or ()
|
||||
transformed_texts: Final = MappingProxyType(
|
||||
{idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)}
|
||||
)
|
||||
transformed_holdback: Final = MappingProxyType(
|
||||
{indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)}
|
||||
)
|
||||
sink.record(transformed_texts, transformed_holdback)
|
||||
returned_texts = guardrailed_inputs.get("texts")
|
||||
# No "texts" key means the guardrail made no change (action NONE): the raw
|
||||
# accumulated text is the guardrailed text. A present-but-shorter list is a
|
||||
# guardrail contract violation; those choices are omitted below (withheld,
|
||||
# not emitted raw) so a malformed response fails closed instead of leaking.
|
||||
if returned_texts is None:
|
||||
returned_texts = texts_to_check
|
||||
elif len(returned_texts) < len(texts_to_check):
|
||||
verbose_proxy_logger.warning(
|
||||
"OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
|
||||
"streaming transform path; withholding the unmatched choices to fail closed.",
|
||||
len(returned_texts),
|
||||
len(texts_to_check),
|
||||
)
|
||||
|
||||
holdback: Final = guardrailed_inputs.get("stream_holdback_chars") or ()
|
||||
transformed_texts: Final = MappingProxyType(
|
||||
{idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)}
|
||||
)
|
||||
transformed_holdback: Final = MappingProxyType(
|
||||
{
|
||||
indices[i]: coerce_stream_holdback_value(holdback[i])
|
||||
for i in range(len(indices))
|
||||
if i < len(holdback)
|
||||
}
|
||||
)
|
||||
sink.record(transformed_texts, transformed_holdback)
|
||||
finally:
|
||||
for key, prior in (
|
||||
("response", prior_response),
|
||||
("responses", prior_responses),
|
||||
("guardrail_choice_index", prior_choice_index),
|
||||
):
|
||||
if prior is missing:
|
||||
resolved_request_data.pop(key, None)
|
||||
else:
|
||||
resolved_request_data[key] = prior
|
||||
|
||||
def get_streaming_scan_key(self, responses_so_far: Sequence[object]) -> StreamingScanKey | None:
|
||||
chunks: Final = tuple(chunk for chunk in responses_so_far if isinstance(chunk, ModelResponseStream))
|
||||
|
|
|
|||
|
|
@ -1437,14 +1437,8 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
assert request_data["metadata"]["trace"] == "retained"
|
||||
if transform:
|
||||
assert request_data["responses"] is chunks
|
||||
restored: Final = request_data["response"]
|
||||
assert isinstance(restored, ModelResponse)
|
||||
assert tuple(choice.index for choice in restored.choices) == (0, 1)
|
||||
else:
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
|
||||
|
|
@ -1679,6 +1673,164 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
assert sent is not None
|
||||
assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_buffered_stream_that_carried_no_choices_is_not_an_error(self) -> None:
|
||||
"""A stream can end carrying only usage, and a policy step still runs over it."""
|
||||
from litellm.types.utils import ModelResponseStream
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[], usage={"prompt_tokens": 3, "completion_tokens": 0,
|
||||
"total_tokens": 3})]
|
||||
result: Final = await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
request_data={},
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert result is chunks
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_round_with_nothing_to_judge_costs_one_call_at_n2(self) -> None:
|
||||
"""Per-choice rounds buy tool-call context. While no call is complete there is none to buy,
|
||||
and charging a vendor once per choice for every sampled round buys nothing."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class CountingGuardrail(CustomGuardrail):
|
||||
calls: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).calls.append(inputs.get("texts"))
|
||||
return inputs
|
||||
|
||||
def choice(index: int, arguments: str) -> StreamingChoices:
|
||||
return StreamingChoices(
|
||||
index=index,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments}},
|
||||
]),
|
||||
finish_reason=None,
|
||||
)
|
||||
|
||||
CountingGuardrail.calls = []
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=[ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"ci')])],
|
||||
guardrail_to_apply=CountingGuardrail(guardrail_name="counting"),
|
||||
litellm_logging_obj=None, request_data={},
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert len(CountingGuardrail.calls) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None:
|
||||
"""Half a JSON object is not a call a guardrail can judge, at any number of choices."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(request_data.get("response"))
|
||||
return inputs
|
||||
|
||||
def choice(index: int, arguments: str, label: str) -> StreamingChoices:
|
||||
return StreamingChoices(
|
||||
index=index,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_" + label, "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=[ModelResponseStream(choices=[
|
||||
choice(0, '{"city": "Pa', "partial"), choice(1, '{"city": "Rome"}', "rome"),
|
||||
])],
|
||||
guardrail_to_apply=RecordingGuardrail(guardrail_name="recording"),
|
||||
litellm_logging_obj=None, request_data={},
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
arguments: Final = tuple(
|
||||
tool_call.function.arguments
|
||||
for response in RecordingGuardrail.seen
|
||||
for single in getattr(response, "choices", None) or ()
|
||||
for tool_call in single.message.tool_calls or ()
|
||||
)
|
||||
assert arguments == ('{"city": "Rome"}',)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_caller_cannot_set_the_choice_index_label(self) -> None:
|
||||
"""The label says which choice the round inspected, so only the round may set it."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class LabelReadingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(request_data.get("guardrail_choice_index"))
|
||||
return inputs
|
||||
|
||||
LabelReadingGuardrail.seen = []
|
||||
request_data: Final[dict[str, object]] = {"model": "gpt-4", "guardrail_choice_index": 7}
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=LabelReadingGuardrail(guardrail_name="label"),
|
||||
litellm_logging_obj=None,
|
||||
request_data=request_data,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert LabelReadingGuardrail.seen == [0]
|
||||
assert request_data["guardrail_choice_index"] == 7
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_transform_round_leaves_the_callers_request_data_as_it_found_it(self) -> None:
|
||||
"""The round owns these keys only while it runs; what it leaves behind is read by later hooks."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
request_data: Final[dict[str, object]] = {"model": "gpt-4"}
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": '{"city": "Paris"}'}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=MockGuardrail(guardrail_name="test"),
|
||||
litellm_logging_obj=None,
|
||||
request_data=request_data,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
assert "guardrail_choice_index" not in request_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unfinished_parallel_call_is_kept_out_of_what_the_guardrail_reads(self) -> None:
|
||||
"""A call that only announced its name must not reach the guardrail, by either route."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue