mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(cohere): parse streamed tool calls with a typed model and map Cohere TOOL_CALL to tool_calls
Read the streamed tool call through a frozen pydantic model instead of casts, which the strict and type-discipline gates refuse. Map Cohere's TOOL_CALL and STOP_SEQUENCE finish reasons, so a stream that reads Cohere's message-end still ends tool_calls. The regression test now drives litellm.acompletion with two parallel streamed calls and checks the rebuilt calls and the final finish reason
This commit is contained in:
parent
a3dd9c1f4f
commit
f6bed1997b
5 changed files with 78 additions and 53 deletions
|
|
@ -242,6 +242,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
|
|||
"compaction": "length",
|
||||
# Cohere
|
||||
"COMPLETE": "stop",
|
||||
"STOP_SEQUENCE": "stop",
|
||||
"TOOL_CALL": "tool_calls",
|
||||
"ERROR_TOXIC": "content_filter",
|
||||
"ERROR": "stop",
|
||||
# HuggingFace / Together AI
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import json
|
||||
from typing import Final, Literal, cast
|
||||
from typing import Final, Literal
|
||||
|
||||
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.llms.cohere import CohereV2StreamedToolCall
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionToolCallChunk,
|
||||
|
|
@ -234,19 +235,13 @@ class CohereV2ModelResponseIterator:
|
|||
return content
|
||||
return ""
|
||||
|
||||
def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk | None:
|
||||
def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
|
||||
"""Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
|
||||
delta: Final = cast("dict[str, dict[str, object]]", chunk.get("delta") or {})
|
||||
raw: Final = delta.get("message", {}).get("tool_calls") or {}
|
||||
tool_call: Final = cast("dict[str, object]", (raw[0] if raw else {}) if isinstance(raw, list) else raw)
|
||||
function: Final = cast("dict[str, str]", tool_call.get("function") or {})
|
||||
tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
|
||||
return {
|
||||
"id": cast("str", tool_call.get("id")),
|
||||
"id": tool_call.id,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": function.get("name"),
|
||||
"arguments": function.get("arguments", ""),
|
||||
},
|
||||
"function": {"name": tool_call.name, "arguments": tool_call.arguments},
|
||||
"index": index,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from typing import Literal
|
||||
|
||||
from pydantic import AliasPath, BaseModel, ConfigDict, Field
|
||||
from typing_extensions import Required, TypedDict
|
||||
|
||||
|
||||
|
|
@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict):
|
|||
message: CohereV2ChatResponseMessage
|
||||
usage: CohereV2ChatResponseUsage
|
||||
logprobs: CohereV2ChatResponseLogProbs
|
||||
|
||||
|
||||
class CohereV2StreamedToolCall(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id"))
|
||||
name: str | None = Field(
|
||||
default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name")
|
||||
)
|
||||
arguments: str = Field(
|
||||
default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments")
|
||||
)
|
||||
|
|
|
|||
|
|
@ -309,6 +309,12 @@ class TestMapFinishReasonCohere:
|
|||
def test_error(self):
|
||||
assert map_finish_reason("ERROR") == "stop"
|
||||
|
||||
def test_tool_call(self):
|
||||
assert map_finish_reason("TOOL_CALL") == "tool_calls"
|
||||
|
||||
def test_stop_sequence(self):
|
||||
assert map_finish_reason("STOP_SEQUENCE") == "stop"
|
||||
|
||||
|
||||
class TestMapFinishReasonHuggingFace:
|
||||
def test_eos_token(self):
|
||||
|
|
|
|||
|
|
@ -665,15 +665,15 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None:
|
|||
assert "invalid request: model 'invalid-model' not found" in error.value.message
|
||||
|
||||
|
||||
def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None:
|
||||
events: Final = [
|
||||
def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]:
|
||||
return (
|
||||
{
|
||||
"type": "tool-call-start",
|
||||
"index": 0,
|
||||
"index": index,
|
||||
"delta": {
|
||||
"message": {
|
||||
"tool_calls": {
|
||||
"id": "get_weather_a1",
|
||||
"id": call_id,
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": ""},
|
||||
}
|
||||
|
|
@ -682,49 +682,58 @@ def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None:
|
|||
},
|
||||
{
|
||||
"type": "tool-call-delta",
|
||||
"index": 0,
|
||||
"index": index,
|
||||
"delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}},
|
||||
},
|
||||
{
|
||||
"type": "tool-call-delta",
|
||||
"index": 0,
|
||||
"delta": {"message": {"tool_calls": {"function": {"arguments": '"Paris"}'}}}},
|
||||
"index": index,
|
||||
"delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}},
|
||||
},
|
||||
{"type": "tool-call-end", "index": 0},
|
||||
{
|
||||
"type": "tool-call-start",
|
||||
"index": 1,
|
||||
"delta": {
|
||||
"message": {
|
||||
"tool_calls": {
|
||||
"id": "get_time_b2",
|
||||
"type": "function",
|
||||
"function": {"name": "get_time", "arguments": ""},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "tool-call-delta",
|
||||
"index": 1,
|
||||
"delta": {"message": {"tool_calls": {"function": {"arguments": "{}"}}}},
|
||||
},
|
||||
]
|
||||
iterator: Final = CohereV2ChatConfig().get_model_response_iterator(
|
||||
streaming_response=iter(json.dumps(event) for event in events), sync_stream=True
|
||||
{"type": "tool-call-end", "index": index},
|
||||
)
|
||||
|
||||
calls: Final[dict[int, dict[str, str]]] = {}
|
||||
for parsed in iterator:
|
||||
tool_use = parsed["tool_use"]
|
||||
if tool_use is None:
|
||||
continue
|
||||
call = calls.setdefault(tool_use["index"], {"id": "", "name": "", "arguments": ""})
|
||||
call["id"] += tool_use["id"] or ""
|
||||
call["name"] += tool_use["function"]["name"] or ""
|
||||
call["arguments"] += tool_use["function"]["arguments"] or ""
|
||||
|
||||
assert calls == {
|
||||
0: {"id": "get_weather_a1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
|
||||
1: {"id": "get_time_b2", "name": "get_time", "arguments": "{}"},
|
||||
}
|
||||
@pytest.mark.usefixtures("_cohere_httpx_transport")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None:
|
||||
stream_events: Final = (
|
||||
{"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}},
|
||||
*_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"),
|
||||
*_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"),
|
||||
{
|
||||
"type": "message-end",
|
||||
"delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}},
|
||||
},
|
||||
)
|
||||
respx_mock.post(COHERE_V2_CHAT_URL).mock(
|
||||
return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
|
||||
)
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model="cohere_chat/v2/command-a-plus-05-2026",
|
||||
messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
|
||||
},
|
||||
}
|
||||
],
|
||||
stream=True,
|
||||
api_key="cohere-test-key",
|
||||
)
|
||||
chunks: Final = [chunk async for chunk in response]
|
||||
rebuilt: Final = litellm.stream_chunk_builder(chunks)
|
||||
|
||||
assert rebuilt is not None
|
||||
assert [
|
||||
(call.id, call.function.name, json.loads(call.function.arguments))
|
||||
for call in rebuilt.choices[0].message.tool_calls
|
||||
] == [
|
||||
("get_weather_paris", "get_weather", {"city": "Paris"}),
|
||||
("get_weather_tokyo", "get_weather", {"city": "Tokyo"}),
|
||||
]
|
||||
assert chunks[-1].choices[0].finish_reason == "tool_calls"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue