fix(cohere): parse streamed tool calls with a typed model and map Cohere TOOL_CALL to tool_calls

Read the streamed tool call through a frozen pydantic model instead of casts,
which the strict and type-discipline gates refuse. Map Cohere's TOOL_CALL and
STOP_SEQUENCE finish reasons, so a stream that reads Cohere's message-end still
ends tool_calls. The regression test now drives litellm.acompletion with two
parallel streamed calls and checks the rebuilt calls and the final finish reason
This commit is contained in:
mateo-berri 2026-10-30 22:02:12 -07:00
parent a3dd9c1f4f
commit f6bed1997b
5 changed files with 78 additions and 53 deletions

View file

@ -242,6 +242,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
"compaction": "length",
# Cohere
"COMPLETE": "stop",
"STOP_SEQUENCE": "stop",
"TOOL_CALL": "tool_calls",
"ERROR_TOXIC": "content_filter",
"ERROR": "stop",
# HuggingFace / Together AI

View file

@ -1,8 +1,9 @@
import json
from typing import Final, Literal, cast
from typing import Final, Literal
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.cohere import CohereV2StreamedToolCall
from litellm.types.llms.openai import AllMessageValues
from litellm.types.utils import (
ChatCompletionToolCallChunk,
@ -234,19 +235,13 @@ class CohereV2ModelResponseIterator:
return content
return ""
def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk | None:
def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
"""Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
delta: Final = cast("dict[str, dict[str, object]]", chunk.get("delta") or {})
raw: Final = delta.get("message", {}).get("tool_calls") or {}
tool_call: Final = cast("dict[str, object]", (raw[0] if raw else {}) if isinstance(raw, list) else raw)
function: Final = cast("dict[str, str]", tool_call.get("function") or {})
tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
return {
"id": cast("str", tool_call.get("id")),
"id": tool_call.id,
"type": "function",
"function": {
"name": function.get("name"),
"arguments": function.get("arguments", ""),
},
"function": {"name": tool_call.name, "arguments": tool_call.arguments},
"index": index,
}

View file

@ -1,5 +1,6 @@
from typing import Literal
from pydantic import AliasPath, BaseModel, ConfigDict, Field
from typing_extensions import Required, TypedDict
@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict):
message: CohereV2ChatResponseMessage
usage: CohereV2ChatResponseUsage
logprobs: CohereV2ChatResponseLogProbs
class CohereV2StreamedToolCall(BaseModel):
model_config = ConfigDict(frozen=True)
id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id"))
name: str | None = Field(
default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name")
)
arguments: str = Field(
default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments")
)

View file

@ -309,6 +309,12 @@ class TestMapFinishReasonCohere:
def test_error(self):
assert map_finish_reason("ERROR") == "stop"
def test_tool_call(self):
assert map_finish_reason("TOOL_CALL") == "tool_calls"
def test_stop_sequence(self):
assert map_finish_reason("STOP_SEQUENCE") == "stop"
class TestMapFinishReasonHuggingFace:
def test_eos_token(self):

View file

@ -665,15 +665,15 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None:
assert "invalid request: model 'invalid-model' not found" in error.value.message
def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None:
events: Final = [
def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]:
return (
{
"type": "tool-call-start",
"index": 0,
"index": index,
"delta": {
"message": {
"tool_calls": {
"id": "get_weather_a1",
"id": call_id,
"type": "function",
"function": {"name": "get_weather", "arguments": ""},
}
@ -682,49 +682,58 @@ def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None:
},
{
"type": "tool-call-delta",
"index": 0,
"index": index,
"delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}},
},
{
"type": "tool-call-delta",
"index": 0,
"delta": {"message": {"tool_calls": {"function": {"arguments": '"Paris"}'}}}},
"index": index,
"delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}},
},
{"type": "tool-call-end", "index": 0},
{
"type": "tool-call-start",
"index": 1,
"delta": {
"message": {
"tool_calls": {
"id": "get_time_b2",
"type": "function",
"function": {"name": "get_time", "arguments": ""},
}
}
},
},
{
"type": "tool-call-delta",
"index": 1,
"delta": {"message": {"tool_calls": {"function": {"arguments": "{}"}}}},
},
]
iterator: Final = CohereV2ChatConfig().get_model_response_iterator(
streaming_response=iter(json.dumps(event) for event in events), sync_stream=True
{"type": "tool-call-end", "index": index},
)
calls: Final[dict[int, dict[str, str]]] = {}
for parsed in iterator:
tool_use = parsed["tool_use"]
if tool_use is None:
continue
call = calls.setdefault(tool_use["index"], {"id": "", "name": "", "arguments": ""})
call["id"] += tool_use["id"] or ""
call["name"] += tool_use["function"]["name"] or ""
call["arguments"] += tool_use["function"]["arguments"] or ""
assert calls == {
0: {"id": "get_weather_a1", "name": "get_weather", "arguments": '{"city": "Paris"}'},
1: {"id": "get_time_b2", "name": "get_time", "arguments": "{}"},
}
@pytest.mark.usefixtures("_cohere_httpx_transport")
@pytest.mark.respx(assert_all_called=True)
async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None:
stream_events: Final = (
{"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}},
*_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"),
*_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"),
{
"type": "message-end",
"delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}},
},
)
respx_mock.post(COHERE_V2_CHAT_URL).mock(
return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode())
)
response: Final = await litellm.acompletion(
model="cohere_chat/v2/command-a-plus-05-2026",
messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
},
}
],
stream=True,
api_key="cohere-test-key",
)
chunks: Final = [chunk async for chunk in response]
rebuilt: Final = litellm.stream_chunk_builder(chunks)
assert rebuilt is not None
assert [
(call.id, call.function.name, json.loads(call.function.arguments))
for call in rebuilt.choices[0].message.tool_calls
] == [
("get_weather_paris", "get_weather", {"city": "Paris"}),
("get_weather_tokyo", "get_weather", {"city": "Tokyo"}),
]
assert chunks[-1].choices[0].finish_reason == "tool_calls"