mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(streaming): coerce a chunk's provider usage model to litellm Usage
`usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's
model does not coerce it. A chunk carrying both choices and usage therefore travels on
with openai's CompletionUsage: callers reading the streamed chunk, and the logging path,
get PromptTokensDetails where every other route hands them PromptTokensDetailsWrapper.
Coerce at the assignment rather than skipping it. Guarding on `isinstance(..., Usage)`
looks equivalent and is not — for two providers this line is the only thing carrying
usage forward:
- azure_text: handle_azure_text_completion_chunk returns no usage key at all, and the
dispatch branch sets none, so skipping drops usage entirely. stream_chunk_builder
then re-derives prompt_tokens with token_counter — 9521 reported becomes 8.
- text-completion-openai: the branch builds a deliberately lossy three-field Usage
with no prompt_tokens_details, so skipping drops cached_tokens and bills them at the
full input rate.
Both are covered by tests that fail under that variant.
Scope, stated precisely: this does NOT change billing on the chat path.
ChunkProcessor._extract_usage_chunk already normalizes a raw provider model into Usage
(9e25dd708f), so prompt_tokens and cached_tokens survive there either way — verified
identical before and after on an openai chunk with cached_tokens=9504. What changes is
the type the caller and the logging path observe, plus the two provider paths above.
Tests are mutation-checked: reverting to the raw assignment fails the two coercion
tests; the isinstance variant fails the azure_text and text-completion-openai tests.
This commit is contained in:
parent
daa665e578
commit
a8320749bb
2 changed files with 148 additions and 1 deletions
|
|
@ -92,6 +92,30 @@ def print_verbose(print_statement: object):
|
|||
pass
|
||||
|
||||
|
||||
def _as_litellm_usage(usage: object) -> Usage:
|
||||
"""Coerce a chunk's usage to litellm's own `Usage`.
|
||||
|
||||
`usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's
|
||||
model (openai's CompletionUsage, say) does not coerce it. The object then travels on
|
||||
as the provider's type: callers reading the streamed chunk, and the logging path, get
|
||||
PromptTokensDetails where every other route gives them PromptTokensDetailsWrapper.
|
||||
|
||||
Coercing rather than skipping the assignment, because for two providers this line is
|
||||
the only thing carrying usage forward: azure_text's chunk handler returns no usage at
|
||||
all, and text-completion-openai builds a deliberately lossy three-field Usage that
|
||||
drops prompt_tokens_details. Guarding the assignment on `isinstance(..., Usage)`
|
||||
silently discards provider-reported tokens on the first and cached tokens on the
|
||||
second.
|
||||
"""
|
||||
if isinstance(usage, Usage):
|
||||
return usage
|
||||
if isinstance(usage, BaseModel):
|
||||
return Usage(**usage.model_dump())
|
||||
if isinstance(usage, dict):
|
||||
return Usage(**usage)
|
||||
return Usage()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderChunkParsed:
|
||||
response_obj: dict[str, object]
|
||||
|
|
@ -1658,7 +1682,7 @@ class CustomStreamWrapper:
|
|||
self.tool_call = True
|
||||
|
||||
if hasattr(chunk, "usage") and chunk.usage is not None:
|
||||
model_response.usage = chunk.usage
|
||||
model_response.usage = _as_litellm_usage(chunk.usage)
|
||||
|
||||
## RETURN ARG
|
||||
result: Final = self.return_processed_chunk_logic(
|
||||
|
|
|
|||
|
|
@ -4917,3 +4917,126 @@ class TestStableStreamingResponseId:
|
|||
)
|
||||
wrapper.response_id = "chatcmpl-from-provider"
|
||||
assert wrapper.model_response_creator().id == "chatcmpl-from-provider"
|
||||
|
||||
|
||||
def _openai_chunk_with_usage(usage):
|
||||
"""An openai chunk carrying BOTH choices and usage — the shape that reaches the
|
||||
assignment under test. A usage-only final chunk returns earlier, via the
|
||||
include_usage path, and never gets here."""
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
|
||||
chunk = ChatCompletionChunk(
|
||||
id="chatcmpl-1",
|
||||
choices=[{"delta": {"content": "hi"}, "finish_reason": None, "index": 0}],
|
||||
created=1,
|
||||
model="gpt-4o",
|
||||
object="chat.completion.chunk",
|
||||
)
|
||||
setattr(chunk, "usage", usage)
|
||||
return chunk
|
||||
|
||||
|
||||
def _completion_usage(prompt_tokens=9521, cached_tokens=9504):
|
||||
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
|
||||
|
||||
return CompletionUsage(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=7,
|
||||
total_tokens=prompt_tokens + 7,
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens),
|
||||
)
|
||||
|
||||
|
||||
def test_provider_usage_model_is_coerced_to_litellm_usage():
|
||||
"""A provider SDK usage model must not travel on as the provider's own type.
|
||||
|
||||
`usage` is not a declared field on ModelResponseStream, so assigning openai's
|
||||
CompletionUsage does not coerce it. The chunk handed to the caller and to logging then
|
||||
carries PromptTokensDetails where every other route gives PromptTokensDetailsWrapper.
|
||||
"""
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
wrapper = litellm.CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="gpt-4o",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
processed = wrapper.chunk_creator(chunk=_openai_chunk_with_usage(_completion_usage()))
|
||||
|
||||
assert isinstance(processed.usage, Usage)
|
||||
assert processed.usage.prompt_tokens == 9521
|
||||
assert processed.usage.prompt_tokens_details.cached_tokens == 9504
|
||||
|
||||
|
||||
def test_text_completion_openai_keeps_cached_tokens():
|
||||
"""text-completion-openai builds a lossy three-field Usage that drops
|
||||
prompt_tokens_details, so this assignment is what carries cached tokens forward.
|
||||
Skipping it on a non-Usage instance bills cached tokens at the full input rate — the
|
||||
exact harm this line exists to prevent."""
|
||||
from openai.types import Completion
|
||||
|
||||
chunk = Completion(
|
||||
id="cmpl-1",
|
||||
choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}],
|
||||
created=1,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
object="text_completion",
|
||||
)
|
||||
setattr(chunk, "usage", _completion_usage())
|
||||
|
||||
wrapper = litellm.CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="text-completion-openai",
|
||||
)
|
||||
processed = wrapper.chunk_creator(chunk=chunk)
|
||||
|
||||
assert processed.usage.prompt_tokens_details.cached_tokens == 9504
|
||||
|
||||
|
||||
def test_azure_text_usage_survives():
|
||||
"""azure_text's chunk handler returns no usage key at all, so this assignment is the
|
||||
ONLY thing putting usage on the chunk. Without it, stream_chunk_builder falls back to
|
||||
token_counter and reports a prompt count three orders of magnitude off."""
|
||||
from openai.types import Completion
|
||||
|
||||
chunk = Completion(
|
||||
id="cmpl-1",
|
||||
choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}],
|
||||
created=1,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
object="text_completion",
|
||||
)
|
||||
setattr(chunk, "usage", _completion_usage())
|
||||
|
||||
wrapper = litellm.CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="gpt-3.5-turbo-instruct",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="azure_text",
|
||||
)
|
||||
processed = wrapper.chunk_creator(chunk=chunk)
|
||||
|
||||
assert processed.usage is not None
|
||||
assert processed.usage.prompt_tokens == 9521
|
||||
|
||||
|
||||
def test_dict_usage_is_coerced():
|
||||
"""Some providers attach a plain dict. Usage() over an unrecognised shape must not
|
||||
raise inside the streaming path."""
|
||||
from litellm.types.utils import Usage
|
||||
|
||||
wrapper = litellm.CustomStreamWrapper(
|
||||
completion_stream=None,
|
||||
model="gpt-4o",
|
||||
logging_obj=MagicMock(),
|
||||
custom_llm_provider="openai",
|
||||
)
|
||||
chunk = _openai_chunk_with_usage({"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7})
|
||||
processed = wrapper.chunk_creator(chunk=chunk)
|
||||
|
||||
assert isinstance(processed.usage, Usage)
|
||||
assert processed.usage.prompt_tokens == 3
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue