mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/litellm_guardrail_choice_isolation' into litellm_guardrail_stream_inspection
This commit is contained in:
commit
5de1fcda40
4 changed files with 367 additions and 31 deletions
|
|
@ -1414,6 +1414,9 @@ class CustomGuardrail(CustomLogger):
|
|||
end_time=end_time,
|
||||
duration=duration,
|
||||
masked_entity_count=masked_entity_count,
|
||||
# Set by callers that inspect one choice at a time, so the per-choice entries of an
|
||||
# n > 1 response can be told apart.
|
||||
choice_index=request_data.get("guardrail_choice_index"),
|
||||
**(tracing_detail or {}),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -669,10 +669,20 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if pre_guardrail_tool_calls and len(model_response.choices) > 1
|
||||
else (model_response,)
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
|
||||
inspection_request_data: Final = request_data if request_data is not None else {}
|
||||
missing: Final = object()
|
||||
prior_response: Final = inspection_request_data.get("response", missing)
|
||||
prior_responses: Final = inspection_request_data.get("responses", missing)
|
||||
prior_choice_index: Final = inspection_request_data.get("guardrail_choice_index", missing)
|
||||
undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
|
||||
try:
|
||||
for inspection_response in inspection_responses:
|
||||
inspection_request_data["response"] = inspection_response
|
||||
inspection_request_data["guardrail_choice_index"] = (
|
||||
inspection_response.choices[0].index if len(inspection_response.choices) == 1 else None
|
||||
)
|
||||
inspection_request_data["responses"] = (
|
||||
[
|
||||
self._narrowed_to_choice(chunk, inspection_response.choices[0].index)
|
||||
|
|
@ -681,16 +691,32 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
if len(inspection_response.choices) == 1
|
||||
else responses_so_far
|
||||
)
|
||||
await self.process_output_response(
|
||||
response=inspection_response,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=inspection_request_data,
|
||||
)
|
||||
try:
|
||||
await self.process_output_response(
|
||||
response=inspection_response,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=inspection_request_data,
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if undeliverable is None:
|
||||
undeliverable = exc
|
||||
finally:
|
||||
inspection_request_data["response"] = model_response
|
||||
inspection_request_data["responses"] = responses_so_far
|
||||
if prior_response is missing:
|
||||
inspection_request_data.pop("response", None)
|
||||
else:
|
||||
inspection_request_data["response"] = prior_response
|
||||
if prior_responses is missing:
|
||||
inspection_request_data.pop("responses", None)
|
||||
else:
|
||||
inspection_request_data["responses"] = prior_responses
|
||||
if prior_choice_index is missing:
|
||||
inspection_request_data.pop("guardrail_choice_index", None)
|
||||
else:
|
||||
inspection_request_data["guardrail_choice_index"] = prior_choice_index
|
||||
if undeliverable is not None:
|
||||
raise undeliverable
|
||||
if not deliver_ended_stream_rewrites:
|
||||
return
|
||||
await self._write_ended_stream_text_rewrites(
|
||||
|
|
@ -705,6 +731,50 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _streamed_choice_indices(responses_so_far: Sequence["ModelResponseStream"]) -> Iterator[int]:
|
||||
for chunk in responses_so_far:
|
||||
for choice in chunk.choices:
|
||||
yield choice.index
|
||||
|
||||
@staticmethod
|
||||
def _streamed_argument_fragments(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
) -> Iterator[tuple[tuple[int, int], str]]:
|
||||
for chunk in responses_so_far:
|
||||
for choice in chunk.choices:
|
||||
for call in choice.delta.tool_calls or ():
|
||||
arguments = getattr(getattr(call, "function", None), "arguments", None)
|
||||
yield (choice.index, call.index or 0), arguments if isinstance(arguments, str) else ""
|
||||
|
||||
@staticmethod
|
||||
def _parses_as_json(arguments: str) -> bool:
|
||||
if not arguments:
|
||||
return False
|
||||
try:
|
||||
json.loads(arguments)
|
||||
except ValueError:
|
||||
return False
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _streamed_tool_arguments_are_complete(responses_so_far: Sequence["ModelResponseStream"]) -> bool:
|
||||
"""Whether every streamed tool call now carries arguments that parse as JSON.
|
||||
|
||||
A call that has announced itself but emitted no argument text yet is still in flight, so a
|
||||
parallel call finishing first does not make the whole set complete.
|
||||
"""
|
||||
fragments: Final = tuple(OpenAIChatCompletionsHandler._streamed_argument_fragments(responses_so_far))
|
||||
if not fragments:
|
||||
return False
|
||||
joined: Final = MappingProxyType(
|
||||
{
|
||||
call: "".join(text for other, text in fragments if other == call)
|
||||
for call in frozenset(call for call, _ in fragments)
|
||||
}
|
||||
)
|
||||
return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values())
|
||||
|
||||
@staticmethod
|
||||
def _rebuild_ended_stream_per_choice(
|
||||
responses_so_far: Sequence["ModelResponseStream"],
|
||||
|
|
@ -824,9 +894,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
|
||||
if self._streamed_tool_call_fingerprints(responses_so_far):
|
||||
assembled: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
resolved_request_data["response"] = assembled
|
||||
if len(assembled.choices) > 1:
|
||||
stream_ended: Final = self._first_choice_has_finished(responses_so_far)
|
||||
streamed_choice_indices: Final = frozenset(self._streamed_choice_indices(responses_so_far))
|
||||
tool_arguments_complete: Final = self._streamed_tool_arguments_are_complete(responses_so_far)
|
||||
assembled: Final = (
|
||||
None
|
||||
if len(streamed_choice_indices) == 1 and not stream_ended and not tool_arguments_complete
|
||||
else self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
|
||||
)
|
||||
if assembled is not None:
|
||||
resolved_request_data["response"] = assembled
|
||||
if assembled is not None and len(assembled.choices) > 1:
|
||||
choice_rounds: Final = tuple(
|
||||
(
|
||||
choice,
|
||||
|
|
@ -835,23 +913,36 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
)
|
||||
for choice in assembled.choices
|
||||
)
|
||||
from litellm.proxy.policy_engine.pipeline_executor import (
|
||||
UndeliverableStreamRewrite,
|
||||
)
|
||||
|
||||
transform_undeliverable: UndeliverableStreamRewrite | None = None # rebind-ok: keeps the first failure
|
||||
try:
|
||||
for choice, choice_sink, choice_chunks in choice_rounds:
|
||||
resolved_request_data["response"] = assembled.model_copy(
|
||||
update=MappingProxyType({"choices": [choice]})
|
||||
)
|
||||
resolved_request_data["responses"] = choice_chunks
|
||||
await self._process_streaming_transform(
|
||||
responses_so_far=choice_chunks,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=resolved_request_data,
|
||||
sink=choice_sink,
|
||||
)
|
||||
resolved_request_data["guardrail_choice_index"] = choice.index
|
||||
try:
|
||||
await self._process_streaming_transform(
|
||||
responses_so_far=choice_chunks,
|
||||
guardrail_to_apply=guardrail_to_apply,
|
||||
litellm_logging_obj=litellm_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
request_data=resolved_request_data,
|
||||
sink=choice_sink,
|
||||
)
|
||||
except UndeliverableStreamRewrite as exc:
|
||||
if transform_undeliverable is None:
|
||||
transform_undeliverable = exc
|
||||
finally:
|
||||
resolved_request_data["response"] = assembled
|
||||
resolved_request_data["responses"] = responses_so_far
|
||||
resolved_request_data.pop("guardrail_choice_index", None)
|
||||
if transform_undeliverable is not None:
|
||||
raise transform_undeliverable
|
||||
merged_texts: Final = MappingProxyType(
|
||||
dict(
|
||||
chain.from_iterable(
|
||||
|
|
@ -868,14 +959,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
)
|
||||
sink.record(merged_texts, merged_holdback)
|
||||
return
|
||||
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
MappingProxyType({"index": index, **converted})
|
||||
for index, tool_call in enumerate(tool_calls)
|
||||
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
|
||||
if assembled is not None and tool_arguments_complete:
|
||||
tool_calls: Final = chain.from_iterable(choice.message.tool_calls or () for choice in assembled.choices)
|
||||
inputs["tool_calls"] = TypeAdapter(list[ChatCompletionToolCallChunk]).validate_python(
|
||||
tuple(
|
||||
MappingProxyType({"index": index, **converted})
|
||||
for index, tool_call in enumerate(tool_calls)
|
||||
if (converted := self._convert_tool_call_to_dict(tool_call)) is not None
|
||||
)
|
||||
)
|
||||
)
|
||||
if responses_so_far and getattr(responses_so_far[0], "model", None):
|
||||
inputs["model"] = responses_so_far[0].model
|
||||
guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
|
||||
|
|
|
|||
|
|
@ -3511,6 +3511,7 @@ AUDIT_GUARDRAIL_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"end_time",
|
||||
"duration",
|
||||
"masked_entity_count",
|
||||
"choice_index",
|
||||
"guardrail_id",
|
||||
"policy_template",
|
||||
"detection_method",
|
||||
|
|
@ -3551,6 +3552,9 @@ class StandardLoggingGuardrailInformation(TypedDict, total=False):
|
|||
}
|
||||
"""
|
||||
|
||||
choice_index: ReadOnly[int | None]
|
||||
"""Which choice of an n > 1 response this entry inspected, when the guardrail ran per choice."""
|
||||
|
||||
guardrail_id: str | None
|
||||
"""Unique identifier for the guardrail configuration, e.g. 'gd-eu-pii-001'"""
|
||||
|
||||
|
|
|
|||
|
|
@ -1437,10 +1437,247 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
assert request_data["metadata"]["trace"] == "retained"
|
||||
assert request_data["responses"] is chunks
|
||||
restored: Final = request_data["response"]
|
||||
assert isinstance(restored, ModelResponse)
|
||||
assert tuple(choice.index for choice in restored.choices) == (0, 1)
|
||||
if transform:
|
||||
assert request_data["responses"] is chunks
|
||||
restored: Final = request_data["response"]
|
||||
assert isinstance(restored, ModelResponse)
|
||||
assert tuple(choice.index for choice in restored.choices) == (0, 1)
|
||||
else:
|
||||
assert "response" not in request_data
|
||||
assert "responses" not in request_data
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_undeliverable_rewrite_still_inspects_the_remaining_choices(self) -> None:
|
||||
"""An undeliverable rewrite on one choice must not leave later choices uninspected.
|
||||
|
||||
The caller releases the original chunks when it catches UndeliverableStreamRewrite, so
|
||||
bailing out of the loop would let a request with n > 1 pair a redactable choice with a
|
||||
prohibited one and have the whole response delivered unchecked.
|
||||
"""
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class UndeliverableOnFirstChoice(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
response: Final = request_data["response"]
|
||||
assert isinstance(response, ModelResponse)
|
||||
index: Final = response.choices[0].index
|
||||
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
|
||||
if index == 0:
|
||||
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
|
||||
return inputs
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("redacted", "prohibited"))])]
|
||||
request_data: Final = {}
|
||||
with pytest.raises(UndeliverableStreamRewrite):
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
|
||||
request_data=request_data, deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_undeliverable_rewrite_in_transform_mode_still_inspects_remaining_choices(self) -> None:
|
||||
"""The transform path runs its own per-choice loop and needs the same rule."""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class UndeliverableOnFirstChoice(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
response: Final = request_data["response"]
|
||||
assert isinstance(response, ModelResponse)
|
||||
index: Final = response.choices[0].index
|
||||
request_data["visited_choices"] = (*request_data.get("visited_choices", ()), index)
|
||||
if index == 0:
|
||||
raise UndeliverableStreamRewrite("undeliverable", "tool_call_mismatch")
|
||||
return inputs
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("redacted", "prohibited"))])]
|
||||
request_data: Final = {}
|
||||
with pytest.raises(UndeliverableStreamRewrite):
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks, guardrail_to_apply=UndeliverableOnFirstChoice(guardrail_name="undeliverable"),
|
||||
request_data=request_data, deliver_ended_stream_rewrites=True,
|
||||
stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert request_data["visited_choices"] == (0, 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_buffered_stream_without_finish_reason_takes_the_ended_path(self) -> None:
|
||||
"""A buffered stream has no finish_reason yet, so the ended path must key off the opt-in.
|
||||
|
||||
Dropping ``deliver_ended_stream_rewrites`` from that condition leaves the buffered
|
||||
caller on the in-flight path, where rewrites are never written back.
|
||||
"""
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class PathProbe(CustomGuardrail):
|
||||
saw_assembled_response: bool = False
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).saw_assembled_response = isinstance(request_data.get("response"), ModelResponse)
|
||||
return inputs
|
||||
|
||||
chunks: Final = [
|
||||
ModelResponseStream(
|
||||
choices=[StreamingChoices(index=0, delta=Delta(content="secret"), finish_reason=None)]
|
||||
)
|
||||
]
|
||||
assert OpenAIChatCompletionsHandler()._first_choice_has_finished(chunks) is False
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=PathProbe(guardrail_name="probe"),
|
||||
litellm_logging_obj=None,
|
||||
deliver_ended_stream_rewrites=True,
|
||||
)
|
||||
assert PathProbe.saw_assembled_response is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_keeps_each_choice_holdback_with_two_choices(self) -> None:
|
||||
"""The per-choice sinks are merged, so an n=2 diff round must keep both holdbacks.
|
||||
|
||||
Reporting an empty holdback here would release text the guardrail asked to withhold.
|
||||
"""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class HoldingGuardrail(CustomGuardrail):
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
texts: Final = inputs.get("texts") or ()
|
||||
return {**inputs, "stream_holdback_chars": [len(text) for text in texts]}
|
||||
|
||||
chunks: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=index, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "contact", "arguments": "{}"},
|
||||
}]), finish_reason="tool_calls",
|
||||
) for index, text in enumerate(("alpha", "bravo-long"))])]
|
||||
sink: Final = StreamTransformSink()
|
||||
await OpenAIChatCompletionsHandler().process_output_streaming_response(
|
||||
responses_so_far=chunks,
|
||||
guardrail_to_apply=HoldingGuardrail(guardrail_name="holding"),
|
||||
litellm_logging_obj=None,
|
||||
stream_transform_sink=sink,
|
||||
)
|
||||
assert sink.holdback_per_choice == {0: len("alpha"), 1: len("bravo-long")}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_withholds_tool_calls_until_their_arguments_are_complete(self) -> None:
|
||||
"""Mid-round the assembled arguments are a truncated JSON fragment, so they are not sent.
|
||||
|
||||
Sending them would hand the guardrail something like `{"ci`, which main never did.
|
||||
"""
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(inputs.get("tool_calls"))
|
||||
return inputs
|
||||
|
||||
def chunk(text: str, arguments: str, finish_reason: str | None) -> ModelResponseStream:
|
||||
return ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content=text, tool_calls=[{
|
||||
"index": 0, "id": "call_0", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments},
|
||||
}]), finish_reason=finish_reason,
|
||||
)])
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
|
||||
partial: Final = [chunk("looking", '{"ci', None)]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=partial, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert RecordingGuardrail.seen == [None]
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
complete: Final = [chunk("looking", '{"ci', None), chunk(" it up", 'ty": "Paris"}', "tool_calls")]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=complete, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
sent: Final = RecordingGuardrail.seen[-1]
|
||||
assert sent is not None
|
||||
assert [call["function"]["arguments"] for call in sent] == ['{"city": "Paris"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_diff_mode_withholds_parallel_tool_calls_until_every_one_is_complete(self) -> None:
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import StreamTransformSink
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
class RecordingGuardrail(CustomGuardrail):
|
||||
seen: list[object] = []
|
||||
|
||||
async def apply_guardrail(
|
||||
self, inputs: GenericGuardrailAPIInputs, request_data: dict[str, object],
|
||||
input_type: Literal["request", "response"], logging_obj: object = None,
|
||||
) -> GenericGuardrailAPIInputs:
|
||||
type(self).seen.append(inputs.get("tool_calls"))
|
||||
return inputs
|
||||
|
||||
def call(index: int, arguments: str) -> dict[str, object]:
|
||||
return {
|
||||
"index": index, "id": f"call_{index}", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments},
|
||||
}
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
guardrail: Final = RecordingGuardrail(guardrail_name="recording")
|
||||
one_still_empty: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0, delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, "")]),
|
||||
finish_reason=None,
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=one_still_empty, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
assert RecordingGuardrail.seen == [None]
|
||||
|
||||
RecordingGuardrail.seen = []
|
||||
both_complete: Final = [ModelResponseStream(choices=[StreamingChoices(
|
||||
index=0,
|
||||
delta=Delta(content="looking", tool_calls=[call(0, '{"city": "Paris"}'), call(1, '{"city": "Rome"}')]),
|
||||
finish_reason="tool_calls",
|
||||
)])]
|
||||
await handler.process_output_streaming_response(
|
||||
responses_so_far=both_complete, guardrail_to_apply=guardrail,
|
||||
litellm_logging_obj=None, stream_transform_sink=StreamTransformSink(),
|
||||
)
|
||||
sent: Final = RecordingGuardrail.seen[-1]
|
||||
assert sent is not None
|
||||
assert [c["function"]["arguments"] for c in sent] == ['{"city": "Paris"}', '{"city": "Rome"}']
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deliver_ended_stream_tool_rewrites_keep_choice_indices(self) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue