fix(streaming): coerce a chunk's provider usage model to litellm Usage

`usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's
model does not coerce it. A chunk carrying both choices and usage therefore travels on
with openai's CompletionUsage: callers reading the streamed chunk, and the logging path,
get PromptTokensDetails where every other route hands them PromptTokensDetailsWrapper.

Coerce at the assignment rather than skipping it. Guarding on `isinstance(..., Usage)`
looks equivalent and is not — for two providers this line is the only thing carrying
usage forward:

  - azure_text: handle_azure_text_completion_chunk returns no usage key at all, and the
    dispatch branch sets none, so skipping drops usage entirely. stream_chunk_builder
    then re-derives prompt_tokens with token_counter — 9521 reported becomes 8.
  - text-completion-openai: the branch builds a deliberately lossy three-field Usage
    with no prompt_tokens_details, so skipping drops cached_tokens and bills them at the
    full input rate.

Both are covered by tests that fail under that variant.

Scope, stated precisely: this does NOT change billing on the chat path.
ChunkProcessor._extract_usage_chunk already normalizes a raw provider model into Usage
(9e25dd708f), so prompt_tokens and cached_tokens survive there either way — verified
identical before and after on an openai chunk with cached_tokens=9504. What changes is
the type the caller and the logging path observe, plus the two provider paths above.

Tests are mutation-checked: reverting to the raw assignment fails the two coercion
tests; the isinstance variant fails the azure_text and text-completion-openai tests.
This commit is contained in:
sami 2026-09-01 12:01:43 +03:00
parent daa665e578
commit a8320749bb
2 changed files with 148 additions and 1 deletions

View file

@ -92,6 +92,30 @@ def print_verbose(print_statement: object):
pass
def _as_litellm_usage(usage: object) -> Usage:
"""Coerce a chunk's usage to litellm's own `Usage`.
`usage` is not a declared field on ModelResponseStream, so assigning a provider SDK's
model (openai's CompletionUsage, say) does not coerce it. The object then travels on
as the provider's type: callers reading the streamed chunk, and the logging path, get
PromptTokensDetails where every other route gives them PromptTokensDetailsWrapper.
Coercing rather than skipping the assignment, because for two providers this line is
the only thing carrying usage forward: azure_text's chunk handler returns no usage at
all, and text-completion-openai builds a deliberately lossy three-field Usage that
drops prompt_tokens_details. Guarding the assignment on `isinstance(..., Usage)`
silently discards provider-reported tokens on the first and cached tokens on the
second.
"""
if isinstance(usage, Usage):
return usage
if isinstance(usage, BaseModel):
return Usage(**usage.model_dump())
if isinstance(usage, dict):
return Usage(**usage)
return Usage()
@dataclass(frozen=True, slots=True)
class _ProviderChunkParsed:
response_obj: dict[str, object]
@ -1658,7 +1682,7 @@ class CustomStreamWrapper:
self.tool_call = True
if hasattr(chunk, "usage") and chunk.usage is not None:
model_response.usage = chunk.usage
model_response.usage = _as_litellm_usage(chunk.usage)
## RETURN ARG
result: Final = self.return_processed_chunk_logic(

View file

@ -4917,3 +4917,126 @@ class TestStableStreamingResponseId:
)
wrapper.response_id = "chatcmpl-from-provider"
assert wrapper.model_response_creator().id == "chatcmpl-from-provider"
def _openai_chunk_with_usage(usage):
"""An openai chunk carrying BOTH choices and usage — the shape that reaches the
assignment under test. A usage-only final chunk returns earlier, via the
include_usage path, and never gets here."""
from openai.types.chat import ChatCompletionChunk
chunk = ChatCompletionChunk(
id="chatcmpl-1",
choices=[{"delta": {"content": "hi"}, "finish_reason": None, "index": 0}],
created=1,
model="gpt-4o",
object="chat.completion.chunk",
)
setattr(chunk, "usage", usage)
return chunk
def _completion_usage(prompt_tokens=9521, cached_tokens=9504):
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
return CompletionUsage(
prompt_tokens=prompt_tokens,
completion_tokens=7,
total_tokens=prompt_tokens + 7,
prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens),
)
def test_provider_usage_model_is_coerced_to_litellm_usage():
"""A provider SDK usage model must not travel on as the provider's own type.
`usage` is not a declared field on ModelResponseStream, so assigning openai's
CompletionUsage does not coerce it. The chunk handed to the caller and to logging then
carries PromptTokensDetails where every other route gives PromptTokensDetailsWrapper.
"""
from litellm.types.utils import Usage
wrapper = litellm.CustomStreamWrapper(
completion_stream=None,
model="gpt-4o",
logging_obj=MagicMock(),
custom_llm_provider="openai",
)
processed = wrapper.chunk_creator(chunk=_openai_chunk_with_usage(_completion_usage()))
assert isinstance(processed.usage, Usage)
assert processed.usage.prompt_tokens == 9521
assert processed.usage.prompt_tokens_details.cached_tokens == 9504
def test_text_completion_openai_keeps_cached_tokens():
"""text-completion-openai builds a lossy three-field Usage that drops
prompt_tokens_details, so this assignment is what carries cached tokens forward.
Skipping it on a non-Usage instance bills cached tokens at the full input rate — the
exact harm this line exists to prevent."""
from openai.types import Completion
chunk = Completion(
id="cmpl-1",
choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}],
created=1,
model="gpt-3.5-turbo-instruct",
object="text_completion",
)
setattr(chunk, "usage", _completion_usage())
wrapper = litellm.CustomStreamWrapper(
completion_stream=None,
model="gpt-3.5-turbo-instruct",
logging_obj=MagicMock(),
custom_llm_provider="text-completion-openai",
)
processed = wrapper.chunk_creator(chunk=chunk)
assert processed.usage.prompt_tokens_details.cached_tokens == 9504
def test_azure_text_usage_survives():
"""azure_text's chunk handler returns no usage key at all, so this assignment is the
ONLY thing putting usage on the chunk. Without it, stream_chunk_builder falls back to
token_counter and reports a prompt count three orders of magnitude off."""
from openai.types import Completion
chunk = Completion(
id="cmpl-1",
choices=[{"text": "hi", "finish_reason": "stop", "index": 0, "logprobs": None}],
created=1,
model="gpt-3.5-turbo-instruct",
object="text_completion",
)
setattr(chunk, "usage", _completion_usage())
wrapper = litellm.CustomStreamWrapper(
completion_stream=None,
model="gpt-3.5-turbo-instruct",
logging_obj=MagicMock(),
custom_llm_provider="azure_text",
)
processed = wrapper.chunk_creator(chunk=chunk)
assert processed.usage is not None
assert processed.usage.prompt_tokens == 9521
def test_dict_usage_is_coerced():
"""Some providers attach a plain dict. Usage() over an unrecognised shape must not
raise inside the streaming path."""
from litellm.types.utils import Usage
wrapper = litellm.CustomStreamWrapper(
completion_stream=None,
model="gpt-4o",
logging_obj=MagicMock(),
custom_llm_provider="openai",
)
chunk = _openai_chunk_with_usage({"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7})
processed = wrapper.chunk_creator(chunk=chunk)
assert isinstance(processed.usage, Usage)
assert processed.usage.prompt_tokens == 3