diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 12ba041aff8..32e6e97570f 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1220,12 +1220,11 @@ class CustomStreamWrapper: else: ## else if model_response_str or ( - completion_obj.get("reasoning_content") is None - and completion_obj.get("tool_calls") is None + completion_obj.get("reasoning_content") is None and completion_obj.get("tool_calls") is None ): - completion_obj["content"] = model_response_str + completion_obj["content"] = model_response_str # rebind-ok: builds delta payload else: - completion_obj["content"] = None + completion_obj["content"] = None # rebind-ok: builds delta payload if self.sent_first_chunk is False: completion_obj["role"] = "assistant" self.sent_first_chunk = True @@ -1367,8 +1366,13 @@ class CustomStreamWrapper: raise StopIteration anthropic_response_obj: Final[GChunk] = cast(GChunk, chunk) completion_obj["content"] = anthropic_response_obj["text"] - if "reasoning_content" in anthropic_response_obj and anthropic_response_obj["reasoning_content"] is not None: - completion_obj["reasoning_content"] = anthropic_response_obj["reasoning_content"] + if ( + "reasoning_content" in anthropic_response_obj + and anthropic_response_obj["reasoning_content"] is not None + ): + completion_obj["reasoning_content"] = anthropic_response_obj[ # rebind-ok: forwards reasoning + "reasoning_content" + ] if anthropic_response_obj["is_finished"]: self.received_finish_reason = anthropic_response_obj["finish_reason"] @@ -1383,9 +1387,9 @@ class CustomStreamWrapper: ) if "tool_calls" in anthropic_response_obj and anthropic_response_obj["tool_calls"] is not None: - completion_obj["tool_calls"] = anthropic_response_obj["tool_calls"] + completion_obj["tool_calls"] = anthropic_response_obj["tool_calls"] # rebind-ok: forwards tool calls elif "tool_use" in anthropic_response_obj and anthropic_response_obj["tool_use"] is not None: - completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]] + completion_obj["tool_calls"] = [anthropic_response_obj["tool_use"]] # rebind-ok: forwards tool calls if ( "provider_specific_fields" in anthropic_response_obj @@ -2793,13 +2797,17 @@ def convert_generic_chunk_to_model_response_stream( ) -> ModelResponseStream: from litellm.types.utils import Delta - tool_calls = chunk.get("tool_calls", None) - if tool_calls is None and chunk.get("tool_use") is not None: - tool_calls = [chunk["tool_use"]] + raw_tool_calls: Final = chunk.get("tool_calls", None) + tool_calls: Final = ( + [chunk["tool_use"]] if raw_tool_calls is None and chunk.get("tool_use") is not None else raw_tool_calls + ) - content = chunk["text"] - if not content and (chunk.get("reasoning_content") is not None or tool_calls is not None): - content = None + raw_content: Final = chunk["text"] + content: Final = ( + None + if not raw_content and (chunk.get("reasoning_content") is not None or tool_calls is not None) + else raw_content + ) model_response_stream: Final = ModelResponseStream( id=str(uuid.uuid4()), diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index a95578a41d7..af9e7bc3c50 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -1,4 +1,5 @@ import json +from collections.abc import Sequence from typing import Final import litellm @@ -31,14 +32,6 @@ class ModelResponseIterator: try: processed_chunk: Final = litellm.ModelResponseStream(**chunk) - text = "" - tool_use: ChatCompletionToolCallChunk | None = None - tool_calls: list[ChatCompletionToolCallChunk] | None = None - reasoning_content: str | None = None - is_finished = False - finish_reason = "" - usage: ChatCompletionUsageBlock | None = None - # Usage-only final chunk (OpenAI ``stream_options.include_usage``) # arrives with an empty ``choices`` list — return usage without # indexing ``choices[0]``. @@ -53,21 +46,16 @@ class ModelResponseIterator: index=0, ) - if processed_chunk.choices[0].delta.content is not None: - text = processed_chunk.choices[0].delta.content + choice_0: Final = processed_chunk.choices[0] + choice_delta: Final = choice_0.delta + text: Final[str] = choice_delta.content if choice_delta.content is not None else "" + reasoning_content: Final[str | None] = getattr(choice_delta, "reasoning_content", None) - delta_reasoning: Final[str | None] = getattr(processed_chunk.choices[0].delta, "reasoning_content", None) - if delta_reasoning is not None: - reasoning_content = delta_reasoning - - if ( - processed_chunk.choices[0].delta.tool_calls is not None - and len(processed_chunk.choices[0].delta.tool_calls) > 0 - ): - tool_calls = [] - for tc in processed_chunk.choices[0].delta.tool_calls: - tc_fn = getattr(tc, "function", None) - fn_chunk = ( + tool_calls_list: list[ChatCompletionToolCallChunk] = [] # mutable-ok: accumulates delta calls + if choice_delta.tool_calls is not None: + for tc in choice_delta.tool_calls: + tc_fn: Final = getattr(tc, "function", None) + fn_chunk: Final = ( ChatCompletionToolCallFunctionChunk( name=getattr(tc_fn, "name", None), arguments=getattr(tc_fn, "arguments", "") or "", @@ -75,38 +63,35 @@ class ModelResponseIterator: if tc_fn is not None else ChatCompletionToolCallFunctionChunk(name=None, arguments="") ) - call_chunk = ChatCompletionToolCallChunk( + call_chunk: Final = ChatCompletionToolCallChunk( id=getattr(tc, "id", None), type="function", function=fn_chunk, index=getattr(tc, "index", 0) if getattr(tc, "index", None) is not None else 0, ) - tool_calls.append(call_chunk) - if tool_calls: - tool_use = tool_calls[0] + tool_calls_list.append(call_chunk) - if processed_chunk.choices[0].finish_reason is not None: - is_finished = True - finish_reason = processed_chunk.choices[0].finish_reason + tool_calls: Final[Sequence[ChatCompletionToolCallChunk] | None] = tool_calls_list or None + tool_use: Final[ChatCompletionToolCallChunk | None] = tool_calls_list[0] if tool_calls_list else None + + is_finished: Final[bool] = choice_0.finish_reason is not None + finish_reason: Final[str] = choice_0.finish_reason or "" usage_chunk: Final[Usage | None] = getattr(processed_chunk, "usage", None) - if usage_chunk is not None: - usage = _usage_block(usage_chunk) + usage: Final[ChatCompletionUsageBlock | None] = ( + _usage_block(usage_chunk) if usage_chunk is not None else None + ) - generic_chunk: Final[GenericStreamingChunk] = GenericStreamingChunk( + return GenericStreamingChunk( text=text, tool_use=tool_use, + tool_calls=tool_calls, + reasoning_content=reasoning_content, is_finished=is_finished, finish_reason=finish_reason, usage=usage, index=0, ) - if reasoning_content is not None: - generic_chunk["reasoning_content"] = reasoning_content - if tool_calls is not None: - generic_chunk["tool_calls"] = tool_calls - - return generic_chunk except json.JSONDecodeError: raise ValueError(f"Failed to decode JSON from chunk: {chunk}") diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 1bdb2919ba7..9a90b884507 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -498,8 +498,8 @@ class ModelInfo(ModelInfoBase, total=False): class GenericStreamingChunk(TypedDict, total=False): text: Required[str] tool_use: ChatCompletionToolCallChunk | None - tool_calls: list[ChatCompletionToolCallChunk] | None - reasoning_content: str | None + tool_calls: ReadOnly[Sequence[ChatCompletionToolCallChunk] | None] + reasoning_content: ReadOnly[str | None] is_finished: Required[bool] finish_reason: Required[str] usage: Required[ChatCompletionUsageBlock | None] diff --git a/tests/unit/llms/databricks/test_databricks_streaming_utils.py b/tests/unit/llms/databricks/test_databricks_streaming_utils.py index 0f52afbab3a..b405505ba52 100644 --- a/tests/unit/llms/databricks/test_databricks_streaming_utils.py +++ b/tests/unit/llms/databricks/test_databricks_streaming_utils.py @@ -182,7 +182,9 @@ async def test_async_stream_skips_blank_sse_lines_and_keeps_the_usage_chunk(): chunks: Final = [chunk async for chunk in ModelResponseIterator(streaming_response=lines(), sync_stream=False)] - assert [(chunk["text"], chunk["finish_reason"], (chunk["usage"] or {}).get("prompt_tokens")) for chunk in chunks] == [ + assert [ + (chunk["text"], chunk["finish_reason"], (chunk["usage"] or {}).get("prompt_tokens")) for chunk in chunks + ] == [ ("144", "", None), ("", "stop", None), ("", "", None), @@ -291,4 +293,3 @@ def test_chunk_parser_extracts_multiple_tool_calls_with_indices(): assert result["tool_calls"][1]["index"] == 1 # Backward compatibility with tool_use assert result["tool_use"]["id"] == "call_1" - diff --git a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py index defa02daad9..c43f91edf9d 100644 --- a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py +++ b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py @@ -222,4 +222,3 @@ def test_real_stream_keeps_reasoning_multi_tool_calls_and_terminal_usage(): assert chunks[4].usage.completion_tokens == 64 assert chunks[4].usage.prompt_tokens == 10 assert chunks[4].usage.completion_tokens_details.reasoning_tokens == 30 -