diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d79d2ba5e3c..1d604d49faf 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -798,8 +798,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id}) @staticmethod - def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices": - kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished] + def _choice_without(choice: "Choices", unfinished: frozenset[tuple[int, str]]) -> "Choices": + kept: Final = [call for call in choice.message.tool_calls or () if (choice.index, call.id) not in unfinished] message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None})) return choice.model_copy(update=MappingProxyType({"message": message})) @@ -811,12 +811,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Rebuilding gives a call that never streamed complete arguments an empty JSON object, which reads to a guardrail as a real call taking no arguments rather than an unfinished one. The - calls are matched by the id they announced, since the rebuilt order need not follow the - order the stream gave them. + calls are matched by the choice they belong to and the id they announced, since the rebuilt + order need not follow the order the stream gave them, and providers number tool calls per + choice, so the same id on another choice is another call. """ joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) unfinished: Final = frozenset( - call_id + (key[0], call_id) for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items() if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, "")) ) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 0e4ca539165..63267791ad9 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1721,6 +1721,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert len(CountingGuardrail.calls) == 1 + def test_an_id_in_flight_on_one_choice_does_not_prune_another(self) -> None: + """Providers number tool calls per choice, so the same id on another choice is another call, + and one still in flight must not take a finished call down with it.""" + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def choice(index: int, arguments: str) -> StreamingChoices: + return StreamingChoices( + index=index, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": "call_shared", "type": "function", + "function": {"name": "weather", "arguments": arguments}}, + ]), + finish_reason="tool_calls", + ) + + handler: Final = OpenAIChatCompletionsHandler() + chunks: Final = [ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"city": "Rome"}')])] + assembled: Final = handler._rebuild_ended_stream_per_choice(chunks, None) + pruned: Final = handler._without_unfinished_tool_calls(assembled, chunks) + assert [ + [call.function.arguments for call in single.message.tool_calls or ()] for single in pruned.choices + ] == [[], ['{"city": "Rome"}']] + @pytest.mark.asyncio async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None: """Half a JSON object is not a call a guardrail can judge, at any number of choices."""