diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index b7c6f805233..0827f6f7b12 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -242,6 +242,8 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = { "compaction": "length", # Cohere "COMPLETE": "stop", + "STOP_SEQUENCE": "stop", + "TOOL_CALL": "tool_calls", "ERROR_TOXIC": "content_filter", "ERROR": "stop", # HuggingFace / Together AI diff --git a/litellm/llms/cohere/common_utils.py b/litellm/llms/cohere/common_utils.py index 0d633e9d60b..e2599b04f86 100644 --- a/litellm/llms/cohere/common_utils.py +++ b/litellm/llms/cohere/common_utils.py @@ -1,8 +1,9 @@ import json -from typing import Final, Literal, cast +from typing import Final, Literal from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.llms.cohere import CohereV2StreamedToolCall from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( ChatCompletionToolCallChunk, @@ -234,19 +235,13 @@ class CohereV2ModelResponseIterator: return content return "" - def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk | None: + def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk: """Parse tool-call-start and tool-call-delta chunks to extract tool calls.""" - delta: Final = cast("dict[str, dict[str, object]]", chunk.get("delta") or {}) - raw: Final = delta.get("message", {}).get("tool_calls") or {} - tool_call: Final = cast("dict[str, object]", (raw[0] if raw else {}) if isinstance(raw, list) else raw) - function: Final = cast("dict[str, str]", tool_call.get("function") or {}) + tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk) return { - "id": cast("str", tool_call.get("id")), + "id": tool_call.id, "type": "function", - "function": { - "name": function.get("name"), - "arguments": function.get("arguments", ""), - }, + "function": {"name": tool_call.name, "arguments": tool_call.arguments}, "index": index, } diff --git a/litellm/types/llms/cohere.py b/litellm/types/llms/cohere.py index 92d1a8a573d..e48ef73b134 100644 --- a/litellm/types/llms/cohere.py +++ b/litellm/types/llms/cohere.py @@ -1,5 +1,6 @@ from typing import Literal +from pydantic import AliasPath, BaseModel, ConfigDict, Field from typing_extensions import Required, TypedDict @@ -98,3 +99,15 @@ class CohereV2ChatResponse(TypedDict): message: CohereV2ChatResponseMessage usage: CohereV2ChatResponseUsage logprobs: CohereV2ChatResponseLogProbs + + +class CohereV2StreamedToolCall(BaseModel): + model_config = ConfigDict(frozen=True) + + id: str | None = Field(default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "id")) + name: str | None = Field( + default=None, validation_alias=AliasPath("delta", "message", "tool_calls", "function", "name") + ) + arguments: str = Field( + default="", validation_alias=AliasPath("delta", "message", "tool_calls", "function", "arguments") + ) diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 6459aa98e70..60906fe5cd4 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -309,6 +309,12 @@ class TestMapFinishReasonCohere: def test_error(self): assert map_finish_reason("ERROR") == "stop" + def test_tool_call(self): + assert map_finish_reason("TOOL_CALL") == "tool_calls" + + def test_stop_sequence(self): + assert map_finish_reason("STOP_SEQUENCE") == "stop" + class TestMapFinishReasonHuggingFace: def test_eos_token(self): diff --git a/tests/unit/llms/cohere/chat/test_cohere_transformation.py b/tests/unit/llms/cohere/chat/test_cohere_transformation.py index 5893f62f1a8..d8d9a048e0e 100644 --- a/tests/unit/llms/cohere/chat/test_cohere_transformation.py +++ b/tests/unit/llms/cohere/chat/test_cohere_transformation.py @@ -665,15 +665,15 @@ def test_cohere_v2_error_handling(respx_mock: respx.MockRouter) -> None: assert "invalid request: model 'invalid-model' not found" in error.value.message -def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None: - events: Final = [ +def _cohere_v2_tool_call_events(index: int, call_id: str, city: str) -> tuple[dict[str, object], ...]: + return ( { "type": "tool-call-start", - "index": 0, + "index": index, "delta": { "message": { "tool_calls": { - "id": "get_weather_a1", + "id": call_id, "type": "function", "function": {"name": "get_weather", "arguments": ""}, } @@ -682,49 +682,58 @@ def test_cohere_v2_stream_tool_calls_are_emitted_with_index() -> None: }, { "type": "tool-call-delta", - "index": 0, + "index": index, "delta": {"message": {"tool_calls": {"function": {"arguments": '{"city": '}}}}, }, { "type": "tool-call-delta", - "index": 0, - "delta": {"message": {"tool_calls": {"function": {"arguments": '"Paris"}'}}}}, + "index": index, + "delta": {"message": {"tool_calls": {"function": {"arguments": json.dumps(city) + "}"}}}}, }, - {"type": "tool-call-end", "index": 0}, - { - "type": "tool-call-start", - "index": 1, - "delta": { - "message": { - "tool_calls": { - "id": "get_time_b2", - "type": "function", - "function": {"name": "get_time", "arguments": ""}, - } - } - }, - }, - { - "type": "tool-call-delta", - "index": 1, - "delta": {"message": {"tool_calls": {"function": {"arguments": "{}"}}}}, - }, - ] - iterator: Final = CohereV2ChatConfig().get_model_response_iterator( - streaming_response=iter(json.dumps(event) for event in events), sync_stream=True + {"type": "tool-call-end", "index": index}, ) - calls: Final[dict[int, dict[str, str]]] = {} - for parsed in iterator: - tool_use = parsed["tool_use"] - if tool_use is None: - continue - call = calls.setdefault(tool_use["index"], {"id": "", "name": "", "arguments": ""}) - call["id"] += tool_use["id"] or "" - call["name"] += tool_use["function"]["name"] or "" - call["arguments"] += tool_use["function"]["arguments"] or "" - assert calls == { - 0: {"id": "get_weather_a1", "name": "get_weather", "arguments": '{"city": "Paris"}'}, - 1: {"id": "get_time_b2", "name": "get_time", "arguments": "{}"}, - } +@pytest.mark.usefixtures("_cohere_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +async def test_cohere_v2_stream_returns_parallel_tool_calls(respx_mock: respx.MockRouter) -> None: + stream_events: Final = ( + {"type": "message-start", "id": "c0a2b6e1", "delta": {"message": {"role": "assistant"}}}, + *_cohere_v2_tool_call_events(0, "get_weather_paris", "Paris"), + *_cohere_v2_tool_call_events(1, "get_weather_tokyo", "Tokyo"), + { + "type": "message-end", + "delta": {"finish_reason": "TOOL_CALL", "usage": {"tokens": {"input_tokens": 40, "output_tokens": 30}}}, + }, + ) + respx_mock.post(COHERE_V2_CHAT_URL).mock( + return_value=httpx.Response(200, content="".join(json.dumps(event) + "\n" for event in stream_events).encode()) + ) + + response: Final = await litellm.acompletion( + model="cohere_chat/v2/command-a-plus-05-2026", + messages=[{"role": "user", "content": "What is the weather in Paris and in Tokyo?"}], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + stream=True, + api_key="cohere-test-key", + ) + chunks: Final = [chunk async for chunk in response] + rebuilt: Final = litellm.stream_chunk_builder(chunks) + + assert rebuilt is not None + assert [ + (call.id, call.function.name, json.loads(call.function.arguments)) + for call in rebuilt.choices[0].message.tool_calls + ] == [ + ("get_weather_paris", "get_weather", {"city": "Paris"}), + ("get_weather_tokyo", "get_weather", {"city": "Tokyo"}), + ] + assert chunks[-1].choices[0].finish_reason == "tool_calls"