diff --git a/litellm/llms/cohere/common_utils.py b/litellm/llms/cohere/common_utils.py index e2599b04f86..65531451d52 100644 --- a/litellm/llms/cohere/common_utils.py +++ b/litellm/llms/cohere/common_utils.py @@ -223,6 +223,7 @@ class CohereV2ModelResponseIterator: self.content_blocks: list = [] self.tool_index = -1 self.json_mode = json_mode + self.tool_call_indexes_with_arguments: frozenset[int] = frozenset() def _parse_content_delta(self, chunk: dict) -> str: """Parse content-delta chunks to extract text.""" @@ -238,6 +239,8 @@ class CohereV2ModelResponseIterator: def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk: """Parse tool-call-start and tool-call-delta chunks to extract tool calls.""" tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk) + if tool_call.arguments: + self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index} return { "id": tool_call.id, "type": "function", @@ -245,6 +248,12 @@ class CohereV2ModelResponseIterator: "index": index, } + def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None: + """Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one.""" + if index in self.tool_call_indexes_with_arguments: + return None + return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index} + def _parse_tool_plan_delta(self, chunk: dict) -> dict | None: """Parse tool-plan-delta events to extract tool plan.""" data: Final = chunk.get("data", {}) @@ -298,6 +307,7 @@ class CohereV2ModelResponseIterator: v2 format: - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text - Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls + - Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason @@ -317,8 +327,12 @@ class CohereV2ModelResponseIterator: # Handle different chunk types if chunk_type == "content-delta": text = self._parse_content_delta(chunk) - elif chunk_type in ("tool-call-start", "tool-call-delta"): - tool_use = self._parse_tool_call_delta(chunk, index) + elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"): + tool_use = ( + self._parse_tool_call_end(index) + if chunk_type == "tool-call-end" + else self._parse_tool_call_delta(chunk, index) + ) elif event_type == "tool-plan-delta": provider_specific_fields = self._parse_tool_plan_delta(chunk) elif event_type == "citation-start": diff --git a/tests/unit/llms/cohere/chat/test_cohere_transformation.py b/tests/unit/llms/cohere/chat/test_cohere_transformation.py index d8d9a048e0e..416fb9885fe 100644 --- a/tests/unit/llms/cohere/chat/test_cohere_transformation.py +++ b/tests/unit/llms/cohere/chat/test_cohere_transformation.py @@ -1,3 +1,4 @@ +import itertools import json from collections.abc import Callable, Iterator from typing import Final @@ -737,3 +738,49 @@ async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.Mo ("get_weather_tokyo", "get_weather", {"city": "Tokyo"}), ] assert chunks[-1].choices[0].finish_reason == "tool_calls" + + +@pytest.mark.usefixtures("_cohere_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +async def test_cohere_v2_stream_closes_argument_less_tool_call_with_empty_object(respx_mock: respx.MockRouter) -> None: + stream_events: Final = ( + {"type": "message-start", "id": "5d1e9f20", "delta": {"message": {"role": "assistant"}}}, + { + "type": "tool-call-start", + "index": 0, + "delta": { + "message": { + "tool_calls": { + "id": "get_server_time_0", + "type": "function", + "function": {"name": "get_server_time", "arguments": ""}, + } + } + }, + }, + {"type": "tool-call-end", "index": 0}, + {"type": "message-end", "delta": {"finish_reason": "TOOL_CALL"}}, + ) + respx_mock.post(COHERE_V2_CHAT_URL).mock( + return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode()) + ) + + response: Final = await litellm.acompletion( + model="cohere_chat/v2/command-a-plus-05-2026", + messages=[{"role": "user", "content": "What time is it on the server?"}], + tools=[ + { + "type": "function", + "function": {"name": "get_server_time", "parameters": {"type": "object", "properties": {}}}, + } + ], + stream=True, + api_key="cohere-test-key", + ) + deltas_per_chunk: Final = [chunk.choices[0].delta.tool_calls or () async for chunk in response] + streamed_calls: Final = tuple(itertools.chain.from_iterable(deltas_per_chunk)) + + assert [(call.id, call.function.name) for call in streamed_calls if call.id] == [ + ("get_server_time_0", "get_server_time") + ] + assert json.loads("".join(call.function.arguments for call in streamed_calls)) == {}