mirror of
https://github.com/BerriAI/litellm.git
synced 2026-08-28 05:25:59 +00:00
fix(usage): keep responses usage SDK-parseable and complete streamed reasoning splits
An unknown reasoning split now falls back to reasoning_tokens=0 in the chat-to-responses usage translation, since the OpenAI SDK requires output_tokens_details with an int reasoning_tokens, and the streaming chunk builder caps the tokenized reasoning estimate at completion_tokens and fills text_tokens with the remainder
This commit is contained in:
parent
46a4eda19e
commit
710ef81a80
5 changed files with 87 additions and 19 deletions
|
|
@ -987,7 +987,12 @@ class ChunkProcessor:
|
|||
returned_usage.completion_tokens_details is not None
|
||||
and returned_usage.completion_tokens_details.reasoning_tokens is None
|
||||
):
|
||||
returned_usage.completion_tokens_details.reasoning_tokens = reasoning_tokens
|
||||
capped_reasoning_tokens: Final = min(max(0, reasoning_tokens), returned_usage.completion_tokens)
|
||||
returned_usage.completion_tokens_details.reasoning_tokens = capped_reasoning_tokens
|
||||
if returned_usage.completion_tokens_details.text_tokens is None:
|
||||
returned_usage.completion_tokens_details.text_tokens = (
|
||||
returned_usage.completion_tokens - capped_reasoning_tokens
|
||||
)
|
||||
if prompt_tokens_details is not None:
|
||||
returned_usage.prompt_tokens_details = prompt_tokens_details
|
||||
|
||||
|
|
|
|||
|
|
@ -2303,18 +2303,19 @@ class LiteLLMCompletionResponsesConfig:
|
|||
# Translate completion_tokens_details to output_tokens_details
|
||||
if hasattr(usage, "completion_tokens_details") and usage.completion_tokens_details is not None:
|
||||
completion_details: Final = usage.completion_tokens_details
|
||||
output_details_dict: Final[dict[str, int]] = {}
|
||||
if hasattr(completion_details, "reasoning_tokens") and completion_details.reasoning_tokens is not None:
|
||||
output_details_dict["reasoning_tokens"] = completion_details.reasoning_tokens
|
||||
|
||||
if hasattr(completion_details, "text_tokens") and completion_details.text_tokens is not None:
|
||||
output_details_dict["text_tokens"] = completion_details.text_tokens
|
||||
|
||||
if hasattr(completion_details, "image_tokens") and completion_details.image_tokens is not None:
|
||||
output_details_dict["image_tokens"] = completion_details.image_tokens
|
||||
|
||||
if output_details_dict:
|
||||
response_usage.output_tokens_details = OutputTokensDetails(**output_details_dict)
|
||||
reasoning_token_count: Final = getattr(completion_details, "reasoning_tokens", None)
|
||||
optional_output_details: Final[dict[str, int]] = {
|
||||
field: value
|
||||
for field, value in (
|
||||
("text_tokens", getattr(completion_details, "text_tokens", None)),
|
||||
("image_tokens", getattr(completion_details, "image_tokens", None)),
|
||||
)
|
||||
if value is not None
|
||||
}
|
||||
response_usage.output_tokens_details = OutputTokensDetails(
|
||||
reasoning_tokens=reasoning_token_count if reasoning_token_count is not None else 0,
|
||||
**optional_output_details,
|
||||
)
|
||||
|
||||
return response_usage
|
||||
|
||||
|
|
|
|||
|
|
@ -1308,3 +1308,37 @@ def test_count_reasoning_tokens_counts_visible_reasoning():
|
|||
)
|
||||
|
||||
assert processor.count_reasoning_tokens(response) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"estimated_reasoning_tokens, expected_reasoning_tokens, expected_text_tokens",
|
||||
[(40, 40, 60), (250, 100, 0)],
|
||||
)
|
||||
def test_calculate_usage_fills_unknown_split_from_reasoning_estimate(
|
||||
estimated_reasoning_tokens, expected_reasoning_tokens, expected_text_tokens
|
||||
):
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper
|
||||
|
||||
chunk = ModelResponseStream(
|
||||
id="chatcmpl-unknown-split",
|
||||
model="claude-opus-4-8",
|
||||
choices=[StreamingChoices(finish_reason="stop", index=0, delta=Delta(content=None, role=None))],
|
||||
usage=Usage(
|
||||
prompt_tokens=50,
|
||||
completion_tokens=100,
|
||||
total_tokens=150,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=None, text_tokens=None),
|
||||
),
|
||||
)
|
||||
processor = ChunkProcessor(chunks=[chunk])
|
||||
|
||||
usage = processor.calculate_usage(
|
||||
chunks=[chunk],
|
||||
model="claude-opus-4-8",
|
||||
completion_output="10",
|
||||
reasoning_tokens=estimated_reasoning_tokens,
|
||||
)
|
||||
|
||||
assert usage.completion_tokens == 100
|
||||
assert usage.completion_tokens_details.reasoning_tokens == expected_reasoning_tokens
|
||||
assert usage.completion_tokens_details.text_tokens == expected_text_tokens
|
||||
|
|
|
|||
|
|
@ -2638,10 +2638,10 @@ class TestUsageTransformation:
|
|||
assert response_usage.output_tokens_details.text_tokens == 50
|
||||
assert response_usage.output_tokens_details.image_tokens == 100
|
||||
|
||||
def test_reasoning_tokens_not_forced_to_zero_when_absent(self):
|
||||
# Regression: previously the else branch wrote reasoning_tokens=0 even when
|
||||
# completion_tokens_details had no reasoning (reasoning_tokens=None). That caused
|
||||
# the proxy to always report reasoning_tokens=0 for non-thinking responses.
|
||||
def test_reasoning_tokens_fall_back_to_zero_when_absent(self):
|
||||
# The OpenAI SDK's ResponseUsage requires output_tokens_details.reasoning_tokens
|
||||
# as an int, so an absent count degrades to 0 on the responses wire instead of
|
||||
# dropping output_tokens_details and breaking SDK clients.
|
||||
usage = Usage(
|
||||
prompt_tokens=10,
|
||||
completion_tokens=50,
|
||||
|
|
@ -2672,7 +2672,8 @@ class TestUsageTransformation:
|
|||
)
|
||||
|
||||
assert response_usage.output_tokens_details is not None
|
||||
assert response_usage.output_tokens_details.reasoning_tokens is None
|
||||
assert response_usage.output_tokens_details.reasoning_tokens == 0
|
||||
assert response_usage.output_tokens_details.text_tokens == 50
|
||||
|
||||
def test_reasoning_tokens_preserved_when_thinking_occurred(self):
|
||||
# Regression: reasoning_tokens must survive the chat->responses translation
|
||||
|
|
|
|||
|
|
@ -297,7 +297,8 @@ def test_transform_usage_with_zero_values():
|
|||
|
||||
cached_tokens=0 is preserved (cache was available; nothing was cached).
|
||||
reasoning_tokens=0 is preserved the same way: an explicit provider-reported
|
||||
zero passes through, while an absent value (None) is omitted.
|
||||
zero passes through, while an absent value (None) falls back to 0 because the
|
||||
Responses API wire contract requires reasoning_tokens as an int.
|
||||
"""
|
||||
completion_response = create_mock_completion_response(
|
||||
model="gpt-4",
|
||||
|
|
@ -321,6 +322,32 @@ def test_transform_usage_with_zero_values():
|
|||
print("✓ Transformation preserves explicit reasoning_tokens=0 and omits absent values")
|
||||
|
||||
|
||||
def test_transform_usage_unknown_reasoning_split_keeps_output_tokens_details():
|
||||
"""
|
||||
An unknown reasoning split (reasoning_tokens=None, text_tokens=None) must still
|
||||
emit output_tokens_details with an integer reasoning_tokens: the OpenAI SDK's
|
||||
ResponseUsage requires the field, so omitting it breaks /v1/responses clients.
|
||||
"""
|
||||
from openai.types.responses.response_usage import (
|
||||
OutputTokensDetails as OpenAISDKOutputTokensDetails,
|
||||
)
|
||||
|
||||
from litellm.types.utils import CompletionTokensDetailsWrapper
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=100,
|
||||
completion_tokens=500,
|
||||
total_tokens=600,
|
||||
completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=None, text_tokens=None),
|
||||
)
|
||||
|
||||
responses_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage(usage)
|
||||
|
||||
assert responses_usage.output_tokens_details is not None
|
||||
assert responses_usage.output_tokens_details.reasoning_tokens == 0
|
||||
OpenAISDKOutputTokensDetails.model_validate(responses_usage.output_tokens_details.model_dump(exclude_none=True))
|
||||
|
||||
|
||||
def test_input_tokens_details_requires_cached_tokens():
|
||||
"""
|
||||
Test that InputTokensDetails has cached_tokens as an int with default value 0.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue