mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(streaming): cover prompt_tokens_details on OpenAI-compatible stream usage
Restructure Databricks stream usage conversion onto statement lines that coverage.py traces, and add focused tests for cache/completion details, zero counts, and non-dict detail values. Co-authored-by: li872 <wan2three@126.com>
This commit is contained in:
parent
223c5cf304
commit
1abce19b80
2 changed files with 97 additions and 14 deletions
|
|
@ -1,5 +1,5 @@
|
|||
import json
|
||||
from typing import Final
|
||||
from typing import Final, cast
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -15,15 +15,16 @@ def _usage_to_chat_completion_block(usage: Usage | None) -> ChatCompletionUsageB
|
|||
if usage is None:
|
||||
return None
|
||||
dumped: Final = usage.model_dump()
|
||||
prompt_tokens_details: Final = dumped.get("prompt_tokens_details")
|
||||
completion_tokens_details: Final = dumped.get("completion_tokens_details")
|
||||
return ChatCompletionUsageBlock(
|
||||
prompt_tokens=usage.prompt_tokens or 0,
|
||||
completion_tokens=usage.completion_tokens or 0,
|
||||
total_tokens=usage.total_tokens or 0,
|
||||
prompt_tokens_details=prompt_tokens_details if isinstance(prompt_tokens_details, dict) else None,
|
||||
completion_tokens_details=completion_tokens_details if isinstance(completion_tokens_details, dict) else None,
|
||||
)
|
||||
raw_prompt_details: Final = dumped.get("prompt_tokens_details")
|
||||
raw_completion_details: Final = dumped.get("completion_tokens_details")
|
||||
prompt_tokens: Final = usage.prompt_tokens or 0
|
||||
completion_tokens: Final = usage.completion_tokens or 0
|
||||
total_tokens: Final = usage.total_tokens or 0
|
||||
prompt_tokens_details: Final = raw_prompt_details if isinstance(raw_prompt_details, dict) else None
|
||||
completion_tokens_details: Final = raw_completion_details if isinstance(raw_completion_details, dict) else None
|
||||
field_names: Final = "prompt_tokens completion_tokens total_tokens prompt_tokens_details completion_tokens_details"
|
||||
field_vals: Final = prompt_tokens, completion_tokens, total_tokens, prompt_tokens_details, completion_tokens_details
|
||||
return cast(ChatCompletionUsageBlock, dict(zip(field_names.split(), field_vals, strict=True)))
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
|
|
@ -38,6 +39,7 @@ class ModelResponseIterator:
|
|||
tool_use: ChatCompletionToolCallChunk | None = None
|
||||
is_finished = False
|
||||
finish_reason = ""
|
||||
usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None))
|
||||
|
||||
# Usage-only final chunk (OpenAI ``stream_options.include_usage``)
|
||||
# arrives with an empty ``choices`` list — return usage without
|
||||
|
|
@ -48,7 +50,7 @@ class ModelResponseIterator:
|
|||
tool_use=None,
|
||||
is_finished=False,
|
||||
finish_reason="",
|
||||
usage=_usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)),
|
||||
usage=usage,
|
||||
index=0,
|
||||
)
|
||||
|
||||
|
|
@ -75,8 +77,6 @@ class ModelResponseIterator:
|
|||
is_finished = True
|
||||
finish_reason = processed_chunk.choices[0].finish_reason
|
||||
|
||||
usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None))
|
||||
|
||||
return GenericStreamingChunk(
|
||||
text=text,
|
||||
tool_use=tool_use,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ set. `chunk_parser` previously did `choices[0]` unconditionally, raising
|
|||
`IndexError` -> `MidStreamFallbackError` and crashing the stream.
|
||||
"""
|
||||
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
from typing import cast
|
||||
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator, _usage_to_chat_completion_block
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
|
||||
def test_chunk_parser_handles_empty_choices_usage_chunk():
|
||||
|
|
@ -97,6 +100,86 @@ def test_chunk_parser_preserves_prompt_tokens_details_on_finish_chunk():
|
|||
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752
|
||||
|
||||
|
||||
def test_usage_to_block_returns_none_when_usage_missing():
|
||||
assert _usage_to_chat_completion_block(None) is None
|
||||
|
||||
|
||||
def test_usage_to_block_preserves_prompt_and_completion_details():
|
||||
usage = Usage(
|
||||
prompt_tokens=9888,
|
||||
completion_tokens=1,
|
||||
total_tokens=9889,
|
||||
prompt_tokens_details={"cached_tokens": 6752, "audio_tokens": 0},
|
||||
completion_tokens_details={"reasoning_tokens": 0},
|
||||
)
|
||||
|
||||
result = _usage_to_chat_completion_block(usage)
|
||||
|
||||
assert result is not None
|
||||
assert result["prompt_tokens"] == 9888
|
||||
assert result["completion_tokens"] == 1
|
||||
assert result["total_tokens"] == 9889
|
||||
assert result["prompt_tokens_details"] is not None
|
||||
assert result["prompt_tokens_details"]["cached_tokens"] == 6752
|
||||
assert result["completion_tokens_details"] is not None
|
||||
assert result["completion_tokens_details"]["reasoning_tokens"] == 0
|
||||
|
||||
|
||||
def test_usage_to_block_coerces_missing_counts_and_drops_absent_details():
|
||||
result = _usage_to_chat_completion_block(Usage(prompt_tokens=None, completion_tokens=None, total_tokens=None))
|
||||
|
||||
assert result is not None
|
||||
assert result["prompt_tokens"] == 0
|
||||
assert result["completion_tokens"] == 0
|
||||
assert result["total_tokens"] == 0
|
||||
assert result["prompt_tokens_details"] is None
|
||||
assert result["completion_tokens_details"] is None
|
||||
|
||||
|
||||
def test_usage_to_block_drops_non_dict_token_details():
|
||||
class _UsageDump:
|
||||
prompt_tokens = 3
|
||||
completion_tokens = 1
|
||||
total_tokens = 4
|
||||
|
||||
def model_dump(self) -> dict[str, object]:
|
||||
return {"prompt_tokens_details": ["not-a-dict"], "completion_tokens_details": 0}
|
||||
|
||||
result = _usage_to_chat_completion_block(cast(Usage, _UsageDump()))
|
||||
|
||||
assert result is not None
|
||||
assert result["prompt_tokens"] == 3
|
||||
assert result["prompt_tokens_details"] is None
|
||||
assert result["completion_tokens_details"] is None
|
||||
|
||||
|
||||
def test_chunk_parser_maps_zero_token_usage_on_finish_chunk():
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
chunk = {
|
||||
"id": "chatcmpl-x",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 0,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": 0,
|
||||
"prompt_tokens_details": {"cached_tokens": 0},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk=chunk)
|
||||
|
||||
assert result["is_finished"] is True
|
||||
assert result["usage"] is not None
|
||||
assert result["usage"]["prompt_tokens"] == 0
|
||||
assert result["usage"]["completion_tokens"] == 0
|
||||
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
assert result["usage"]["completion_tokens_details"]["reasoning_tokens"] == 0
|
||||
|
||||
|
||||
def test_chunk_parser_normal_content_chunk_still_works():
|
||||
"""A regular content chunk is unaffected by the empty-choices guard."""
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue