diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index d8ce9fdf95e..8ad7afe71af 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -1,5 +1,5 @@ import json -from typing import Final +from typing import Final, cast import litellm from litellm import verbose_logger @@ -15,15 +15,16 @@ def _usage_to_chat_completion_block(usage: Usage | None) -> ChatCompletionUsageB if usage is None: return None dumped: Final = usage.model_dump() - prompt_tokens_details: Final = dumped.get("prompt_tokens_details") - completion_tokens_details: Final = dumped.get("completion_tokens_details") - return ChatCompletionUsageBlock( - prompt_tokens=usage.prompt_tokens or 0, - completion_tokens=usage.completion_tokens or 0, - total_tokens=usage.total_tokens or 0, - prompt_tokens_details=prompt_tokens_details if isinstance(prompt_tokens_details, dict) else None, - completion_tokens_details=completion_tokens_details if isinstance(completion_tokens_details, dict) else None, - ) + raw_prompt_details: Final = dumped.get("prompt_tokens_details") + raw_completion_details: Final = dumped.get("completion_tokens_details") + prompt_tokens: Final = usage.prompt_tokens or 0 + completion_tokens: Final = usage.completion_tokens or 0 + total_tokens: Final = usage.total_tokens or 0 + prompt_tokens_details: Final = raw_prompt_details if isinstance(raw_prompt_details, dict) else None + completion_tokens_details: Final = raw_completion_details if isinstance(raw_completion_details, dict) else None + field_names: Final = "prompt_tokens completion_tokens total_tokens prompt_tokens_details completion_tokens_details" + field_vals: Final = prompt_tokens, completion_tokens, total_tokens, prompt_tokens_details, completion_tokens_details + return cast(ChatCompletionUsageBlock, dict(zip(field_names.split(), field_vals, strict=True))) class ModelResponseIterator: @@ -38,6 +39,7 @@ class ModelResponseIterator: tool_use: ChatCompletionToolCallChunk | None = None is_finished = False finish_reason = "" + usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)) # Usage-only final chunk (OpenAI ``stream_options.include_usage``) # arrives with an empty ``choices`` list — return usage without @@ -48,7 +50,7 @@ class ModelResponseIterator: tool_use=None, is_finished=False, finish_reason="", - usage=_usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)), + usage=usage, index=0, ) @@ -75,8 +77,6 @@ class ModelResponseIterator: is_finished = True finish_reason = processed_chunk.choices[0].finish_reason - usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)) - return GenericStreamingChunk( text=text, tool_use=tool_use, diff --git a/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py b/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py index 448f91078bf..7271ee91b7e 100644 --- a/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py +++ b/tests/test_litellm/llms/databricks/test_databricks_streaming_utils.py @@ -7,7 +7,10 @@ set. `chunk_parser` previously did `choices[0]` unconditionally, raising `IndexError` -> `MidStreamFallbackError` and crashing the stream. """ -from litellm.llms.databricks.streaming_utils import ModelResponseIterator +from typing import cast + +from litellm.llms.databricks.streaming_utils import ModelResponseIterator, _usage_to_chat_completion_block +from litellm.types.utils import Usage def test_chunk_parser_handles_empty_choices_usage_chunk(): @@ -97,6 +100,86 @@ def test_chunk_parser_preserves_prompt_tokens_details_on_finish_chunk(): assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752 +def test_usage_to_block_returns_none_when_usage_missing(): + assert _usage_to_chat_completion_block(None) is None + + +def test_usage_to_block_preserves_prompt_and_completion_details(): + usage = Usage( + prompt_tokens=9888, + completion_tokens=1, + total_tokens=9889, + prompt_tokens_details={"cached_tokens": 6752, "audio_tokens": 0}, + completion_tokens_details={"reasoning_tokens": 0}, + ) + + result = _usage_to_chat_completion_block(usage) + + assert result is not None + assert result["prompt_tokens"] == 9888 + assert result["completion_tokens"] == 1 + assert result["total_tokens"] == 9889 + assert result["prompt_tokens_details"] is not None + assert result["prompt_tokens_details"]["cached_tokens"] == 6752 + assert result["completion_tokens_details"] is not None + assert result["completion_tokens_details"]["reasoning_tokens"] == 0 + + +def test_usage_to_block_coerces_missing_counts_and_drops_absent_details(): + result = _usage_to_chat_completion_block(Usage(prompt_tokens=None, completion_tokens=None, total_tokens=None)) + + assert result is not None + assert result["prompt_tokens"] == 0 + assert result["completion_tokens"] == 0 + assert result["total_tokens"] == 0 + assert result["prompt_tokens_details"] is None + assert result["completion_tokens_details"] is None + + +def test_usage_to_block_drops_non_dict_token_details(): + class _UsageDump: + prompt_tokens = 3 + completion_tokens = 1 + total_tokens = 4 + + def model_dump(self) -> dict[str, object]: + return {"prompt_tokens_details": ["not-a-dict"], "completion_tokens_details": 0} + + result = _usage_to_chat_completion_block(cast(Usage, _UsageDump())) + + assert result is not None + assert result["prompt_tokens"] == 3 + assert result["prompt_tokens_details"] is None + assert result["completion_tokens_details"] is None + + +def test_chunk_parser_maps_zero_token_usage_on_finish_chunk(): + iterator = ModelResponseIterator(streaming_response=None, sync_stream=True) + chunk = { + "id": "chatcmpl-x", + "object": "chat.completion.chunk", + "created": 1, + "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0, + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + } + + result = iterator.chunk_parser(chunk=chunk) + + assert result["is_finished"] is True + assert result["usage"] is not None + assert result["usage"]["prompt_tokens"] == 0 + assert result["usage"]["completion_tokens"] == 0 + assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 0 + assert result["usage"]["completion_tokens_details"]["reasoning_tokens"] == 0 + + def test_chunk_parser_normal_content_chunk_still_works(): """A regular content chunk is unaffected by the empty-choices guard.""" iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)