mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(cohere): close a streamed tool call that sent no arguments with an empty JSON object
This commit is contained in:
parent
f6bed1997b
commit
54a9233b65
2 changed files with 63 additions and 2 deletions
|
|
@ -223,6 +223,7 @@ class CohereV2ModelResponseIterator:
|
|||
self.content_blocks: list = []
|
||||
self.tool_index = -1
|
||||
self.json_mode = json_mode
|
||||
self.tool_call_indexes_with_arguments: frozenset[int] = frozenset()
|
||||
|
||||
def _parse_content_delta(self, chunk: dict) -> str:
|
||||
"""Parse content-delta chunks to extract text."""
|
||||
|
|
@ -238,6 +239,8 @@ class CohereV2ModelResponseIterator:
|
|||
def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
|
||||
"""Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
|
||||
tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
|
||||
if tool_call.arguments:
|
||||
self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index}
|
||||
return {
|
||||
"id": tool_call.id,
|
||||
"type": "function",
|
||||
|
|
@ -245,6 +248,12 @@ class CohereV2ModelResponseIterator:
|
|||
"index": index,
|
||||
}
|
||||
|
||||
def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None:
|
||||
"""Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one."""
|
||||
if index in self.tool_call_indexes_with_arguments:
|
||||
return None
|
||||
return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index}
|
||||
|
||||
def _parse_tool_plan_delta(self, chunk: dict) -> dict | None:
|
||||
"""Parse tool-plan-delta events to extract tool plan."""
|
||||
data: Final = chunk.get("data", {})
|
||||
|
|
@ -298,6 +307,7 @@ class CohereV2ModelResponseIterator:
|
|||
v2 format:
|
||||
- Content: chunk.type == "content-delta" -> chunk.delta.message.content.text
|
||||
- Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls
|
||||
- Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none
|
||||
- Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan
|
||||
- Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations
|
||||
- Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason
|
||||
|
|
@ -317,8 +327,12 @@ class CohereV2ModelResponseIterator:
|
|||
# Handle different chunk types
|
||||
if chunk_type == "content-delta":
|
||||
text = self._parse_content_delta(chunk)
|
||||
elif chunk_type in ("tool-call-start", "tool-call-delta"):
|
||||
tool_use = self._parse_tool_call_delta(chunk, index)
|
||||
elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"):
|
||||
tool_use = (
|
||||
self._parse_tool_call_end(index)
|
||||
if chunk_type == "tool-call-end"
|
||||
else self._parse_tool_call_delta(chunk, index)
|
||||
)
|
||||
elif event_type == "tool-plan-delta":
|
||||
provider_specific_fields = self._parse_tool_plan_delta(chunk)
|
||||
elif event_type == "citation-start":
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import itertools
|
||||
import json
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Final
|
||||
|
|
@ -737,3 +738,49 @@ async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.Mo
|
|||
("get_weather_tokyo", "get_weather", {"city": "Tokyo"}),
|
||||
]
|
||||
assert chunks[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_cohere_httpx_transport")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
async def test_cohere_v2_stream_closes_argument_less_tool_call_with_empty_object(respx_mock: respx.MockRouter) -> None:
|
||||
stream_events: Final = (
|
||||
{"type": "message-start", "id": "5d1e9f20", "delta": {"message": {"role": "assistant"}}},
|
||||
{
|
||||
"type": "tool-call-start",
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {
|
||||
"tool_calls": {
|
||||
"id": "get_server_time_0",
|
||||
"type": "function",
|
||||
"function": {"name": "get_server_time", "arguments": ""},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{"type": "tool-call-end", "index": 0},
|
||||
{"type": "message-end", "delta": {"finish_reason": "TOOL_CALL"}},
|
||||
)
|
||||
respx_mock.post(COHERE_V2_CHAT_URL).mock(
|
||||
return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
|
||||
)
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model="cohere_chat/v2/command-a-plus-05-2026",
|
||||
messages=[{"role": "user", "content": "What time is it on the server?"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_server_time", "parameters": {"type": "object", "properties": {}}},
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
api_key="cohere-test-key",
|
||||
)
|
||||
deltas_per_chunk: Final = [chunk.choices[0].delta.tool_calls or () async for chunk in response]
|
||||
streamed_calls: Final = tuple(itertools.chain.from_iterable(deltas_per_chunk))
|
||||
|
||||
assert [(call.id, call.function.name) for call in streamed_calls if call.id] == [
|
||||
("get_server_time_0", "get_server_time")
|
||||
]
|
||||
assert json.loads("".join(call.function.arguments for call in streamed_calls)) == {}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue