mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(streaming): preserve prompt_tokens_details on OpenAI-compatible stream usage
OpenAI-compatible stream parsers copied only prompt/completion/total token counts, so Anthropic /v1/messages streaming reported cache_read_input_tokens 0.
This commit is contained in:
parent
30f33a949b
commit
223c5cf304
5 changed files with 171 additions and 23 deletions
|
|
@ -1529,11 +1529,7 @@ class CustomStreamWrapper:
|
|||
setattr(
|
||||
model_response,
|
||||
"usage",
|
||||
litellm.Usage(
|
||||
prompt_tokens=response_obj["usage"].get("prompt_tokens", None) or None,
|
||||
completion_tokens=response_obj["usage"].get("completion_tokens", None) or None,
|
||||
total_tokens=response_obj["usage"].get("total_tokens", None) or None,
|
||||
),
|
||||
litellm.Usage(**response_obj["usage"]),
|
||||
)
|
||||
elif isinstance(response_obj["usage"], Usage):
|
||||
setattr(
|
||||
|
|
|
|||
|
|
@ -11,6 +11,21 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.utils import GenericStreamingChunk, Usage
|
||||
|
||||
|
||||
def _usage_to_chat_completion_block(usage: Usage | None) -> ChatCompletionUsageBlock | None:
|
||||
if usage is None:
|
||||
return None
|
||||
dumped: Final = usage.model_dump()
|
||||
prompt_tokens_details: Final = dumped.get("prompt_tokens_details")
|
||||
completion_tokens_details: Final = dumped.get("completion_tokens_details")
|
||||
return ChatCompletionUsageBlock(
|
||||
prompt_tokens=usage.prompt_tokens or 0,
|
||||
completion_tokens=usage.completion_tokens or 0,
|
||||
total_tokens=usage.total_tokens or 0,
|
||||
prompt_tokens_details=prompt_tokens_details if isinstance(prompt_tokens_details, dict) else None,
|
||||
completion_tokens_details=completion_tokens_details if isinstance(completion_tokens_details, dict) else None,
|
||||
)
|
||||
|
||||
|
||||
class ModelResponseIterator:
|
||||
def __init__(self, streaming_response, sync_stream: bool):
|
||||
self.streaming_response = streaming_response
|
||||
|
|
@ -23,27 +38,17 @@ class ModelResponseIterator:
|
|||
tool_use: ChatCompletionToolCallChunk | None = None
|
||||
is_finished = False
|
||||
finish_reason = ""
|
||||
usage: ChatCompletionUsageBlock | None = None
|
||||
|
||||
# Usage-only final chunk (OpenAI ``stream_options.include_usage``)
|
||||
# arrives with an empty ``choices`` list — return usage without
|
||||
# indexing ``choices[0]``.
|
||||
if len(processed_chunk.choices) == 0:
|
||||
final_usage: Final = getattr(processed_chunk, "usage", None)
|
||||
return GenericStreamingChunk(
|
||||
text="",
|
||||
tool_use=None,
|
||||
is_finished=False,
|
||||
finish_reason="",
|
||||
usage=(
|
||||
ChatCompletionUsageBlock(
|
||||
prompt_tokens=final_usage.prompt_tokens or 0,
|
||||
completion_tokens=final_usage.completion_tokens or 0,
|
||||
total_tokens=final_usage.total_tokens or 0,
|
||||
)
|
||||
if final_usage is not None
|
||||
else None
|
||||
),
|
||||
usage=_usage_to_chat_completion_block(getattr(processed_chunk, "usage", None)),
|
||||
index=0,
|
||||
)
|
||||
|
||||
|
|
@ -70,13 +75,7 @@ class ModelResponseIterator:
|
|||
is_finished = True
|
||||
finish_reason = processed_chunk.choices[0].finish_reason
|
||||
|
||||
usage_chunk: Final[Usage | None] = getattr(processed_chunk, "usage", None)
|
||||
if usage_chunk is not None:
|
||||
usage = ChatCompletionUsageBlock(
|
||||
prompt_tokens=usage_chunk.prompt_tokens,
|
||||
completion_tokens=usage_chunk.completion_tokens,
|
||||
total_tokens=usage_chunk.total_tokens,
|
||||
)
|
||||
usage: Final = _usage_to_chat_completion_block(getattr(processed_chunk, "usage", None))
|
||||
|
||||
return GenericStreamingChunk(
|
||||
text=text,
|
||||
|
|
|
|||
|
|
@ -2511,6 +2511,70 @@ def test_usage_only_chunk_not_dropped_when_finish_reason_already_set(
|
|||
assert result.usage is not None
|
||||
|
||||
|
||||
def test_chunk_creator_preserves_dict_usage_prompt_cache_details(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
from types import SimpleNamespace
|
||||
|
||||
initialized_custom_stream_wrapper.custom_llm_provider = "openai"
|
||||
initialized_custom_stream_wrapper.send_stream_usage = True
|
||||
|
||||
chunk = SimpleNamespace(
|
||||
id="chatcmpl-cache",
|
||||
choices=[],
|
||||
system_fingerprint=None,
|
||||
usage={
|
||||
"prompt_tokens": 9888,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 9889,
|
||||
"prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
)
|
||||
|
||||
result = initialized_custom_stream_wrapper.chunk_creator(chunk=chunk)
|
||||
|
||||
assert result is not None
|
||||
assert result.usage is not None
|
||||
assert result.usage.prompt_tokens == 9888
|
||||
assert result.usage.prompt_tokens_details is not None
|
||||
assert result.usage.prompt_tokens_details.cached_tokens == 6752
|
||||
assert result.usage.completion_tokens_details is not None
|
||||
assert result.usage.completion_tokens_details.reasoning_tokens == 0
|
||||
|
||||
|
||||
def test_chunk_creator_preserves_openai_like_generic_usage_cache_details(
|
||||
initialized_custom_stream_wrapper: CustomStreamWrapper,
|
||||
):
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
|
||||
initialized_custom_stream_wrapper.custom_llm_provider = "openai_like"
|
||||
initialized_custom_stream_wrapper.received_finish_reason = "stop"
|
||||
parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser(
|
||||
chunk={
|
||||
"id": "chatcmpl-cache",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 9888,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 9889,
|
||||
"prompt_tokens_details": {"cached_tokens": 6752},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
result = initialized_custom_stream_wrapper.chunk_creator(chunk=parsed)
|
||||
|
||||
assert result is not None
|
||||
assert result.usage is not None
|
||||
assert result.usage.prompt_tokens_details is not None
|
||||
assert result.usage.prompt_tokens_details.cached_tokens == 6752
|
||||
|
||||
|
||||
def _run_dispatch(wrapper: CustomStreamWrapper, chunk):
|
||||
model_response = wrapper.model_response_creator()
|
||||
completion_obj = {"content": ""}
|
||||
|
|
|
|||
|
|
@ -3253,6 +3253,46 @@ def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_prom
|
|||
assert message_delta["usage"]["cache_creation_input_tokens"] == 20
|
||||
|
||||
|
||||
def test_openai_like_stream_usage_chunk_keeps_cache_read_in_anthropic_delta():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.llms.databricks.streaming_utils import ModelResponseIterator
|
||||
|
||||
parsed = ModelResponseIterator(streaming_response=None, sync_stream=True).chunk_parser(
|
||||
chunk={
|
||||
"id": "chatcmpl-cache",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 120,
|
||||
"completion_tokens": 50,
|
||||
"total_tokens": 170,
|
||||
"prompt_tokens_details": {"cached_tokens": 30, "audio_tokens": 0},
|
||||
},
|
||||
}
|
||||
)
|
||||
wrapper = CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="m",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="openai_like",
|
||||
)
|
||||
wrapper.received_finish_reason = "stop"
|
||||
usage_chunk = wrapper.chunk_creator(chunk=parsed)
|
||||
assert usage_chunk is not None
|
||||
assert usage_chunk.usage is not None
|
||||
|
||||
adapter = LiteLLMAnthropicMessagesAdapter()
|
||||
usage_delta = adapter._translate_openai_usage_to_anthropic_usage_delta(usage_chunk.usage)
|
||||
|
||||
assert usage_delta["input_tokens"] == 90
|
||||
assert usage_delta["output_tokens"] == 50
|
||||
assert usage_delta["cache_read_input_tokens"] == 30
|
||||
|
||||
|
||||
def test_translate_streaming_openai_response_to_anthropic_cache_tokens_from_hidden_params_usage():
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,55 @@ def test_chunk_parser_empty_choices_without_usage():
|
|||
assert result["usage"] is None
|
||||
|
||||
|
||||
def test_chunk_parser_preserves_prompt_tokens_details_on_usage_only_chunk():
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
usage_only_chunk = {
|
||||
"id": "chatcmpl-x",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 9888,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 9889,
|
||||
"prompt_tokens_details": {"cached_tokens": 6752, "audio_tokens": 0},
|
||||
"completion_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk=usage_only_chunk)
|
||||
|
||||
assert result["usage"] is not None
|
||||
assert result["usage"]["prompt_tokens"] == 9888
|
||||
assert result["usage"]["completion_tokens"] == 1
|
||||
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752
|
||||
assert result["usage"]["completion_tokens_details"]["reasoning_tokens"] == 0
|
||||
|
||||
|
||||
def test_chunk_parser_preserves_prompt_tokens_details_on_finish_chunk():
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
chunk = {
|
||||
"id": "chatcmpl-x",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "m",
|
||||
"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
|
||||
"usage": {
|
||||
"prompt_tokens": 9888,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 9889,
|
||||
"prompt_tokens_details": {"cached_tokens": 6752},
|
||||
},
|
||||
}
|
||||
|
||||
result = iterator.chunk_parser(chunk=chunk)
|
||||
|
||||
assert result["is_finished"] is True
|
||||
assert result["usage"] is not None
|
||||
assert result["usage"]["prompt_tokens_details"]["cached_tokens"] == 6752
|
||||
|
||||
|
||||
def test_chunk_parser_normal_content_chunk_still_works():
|
||||
"""A regular content chunk is unaffected by the empty-choices guard."""
|
||||
iterator = ModelResponseIterator(streaming_response=None, sync_stream=True)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue