mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
fix(realtime): store text_tokens without the nested reasoning share
The realtime usage writer passed the provider's output_token_details through as sent, so spend logs and callbacks kept a text_tokens that still contained reasoning_tokens while every other completion_tokens_details producer stores the partitioned share. The writer now applies the same rule the cost calculator uses, moved to litellm/types/utils.py so both read one definition, and the calculator keeps it for usage objects that arrive nested from elsewhere
This commit is contained in:
parent
6294367920
commit
e1d900d1c2
5 changed files with 123 additions and 16 deletions
|
|
@ -26,6 +26,7 @@ from litellm.types.utils import (
|
|||
PromptTokensDetailsWrapper,
|
||||
ServiceTier,
|
||||
Usage,
|
||||
text_tokens_without_nested_reasoning,
|
||||
)
|
||||
from litellm.utils import get_model_info
|
||||
|
||||
|
|
@ -852,17 +853,6 @@ class CompletionTokensDetailsResult(TypedDict):
|
|||
video_tokens: int
|
||||
|
||||
|
||||
def _text_tokens_without_nested_reasoning(
|
||||
completion_tokens: int,
|
||||
text_tokens: int,
|
||||
reasoning_tokens: int,
|
||||
other_modality_tokens: int,
|
||||
) -> int:
|
||||
reported_total: Final = text_tokens + reasoning_tokens + other_modality_tokens
|
||||
nested_reasoning_tokens: Final = min(reasoning_tokens, text_tokens, max(reported_total - completion_tokens, 0))
|
||||
return text_tokens - nested_reasoning_tokens
|
||||
|
||||
|
||||
def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
|
||||
audio_tokens: Final = (
|
||||
cast(
|
||||
|
|
@ -893,7 +883,7 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
|
|||
or 0
|
||||
)
|
||||
video_tokens: Final = _coerce_token_count(getattr(usage.completion_tokens_details, "video_tokens", 0))
|
||||
text_tokens: Final = _text_tokens_without_nested_reasoning(
|
||||
text_tokens: Final = text_tokens_without_nested_reasoning(
|
||||
completion_tokens=usage.completion_tokens,
|
||||
text_tokens=reported_text_tokens,
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
|
|
|
|||
|
|
@ -25,9 +25,15 @@ from litellm.types.utils import (
|
|||
PromptTokensDetailsWrapper,
|
||||
SpecialEnums,
|
||||
Usage,
|
||||
text_tokens_without_nested_reasoning,
|
||||
)
|
||||
|
||||
|
||||
def _output_token_detail(details: object, field: str) -> int | None:
|
||||
value: Final = getattr(details, field, None)
|
||||
return value if isinstance(value, int) else None
|
||||
|
||||
|
||||
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
|
||||
return isinstance(value, list)
|
||||
|
||||
|
|
@ -1134,11 +1140,22 @@ class ResponseAPILoggingUtils:
|
|||
response_api_usage, "output_tokens_details", None
|
||||
)
|
||||
if output_tokens_details:
|
||||
reasoning_tokens: Final = _output_token_detail(output_tokens_details, "reasoning_tokens")
|
||||
image_tokens: Final = _output_token_detail(output_tokens_details, "image_tokens")
|
||||
audio_tokens: Final = _output_token_detail(output_tokens_details, "audio_tokens")
|
||||
reported_text_tokens: Final = _output_token_detail(output_tokens_details, "text_tokens")
|
||||
completion_tokens_details = CompletionTokensDetailsWrapper(
|
||||
reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None),
|
||||
image_tokens=getattr(output_tokens_details, "image_tokens", None),
|
||||
text_tokens=getattr(output_tokens_details, "text_tokens", None),
|
||||
audio_tokens=getattr(output_tokens_details, "audio_tokens", None),
|
||||
reasoning_tokens=reasoning_tokens,
|
||||
image_tokens=image_tokens,
|
||||
text_tokens=None
|
||||
if reported_text_tokens is None
|
||||
else text_tokens_without_nested_reasoning(
|
||||
completion_tokens=completion_tokens,
|
||||
text_tokens=reported_text_tokens,
|
||||
reasoning_tokens=reasoning_tokens or 0,
|
||||
other_modality_tokens=(audio_tokens or 0) + (image_tokens or 0),
|
||||
),
|
||||
audio_tokens=audio_tokens,
|
||||
)
|
||||
|
||||
extra_usage_fields: Final = {
|
||||
|
|
|
|||
|
|
@ -1622,6 +1622,17 @@ class Choices(SafeAttributeModel, OpenAIObject):
|
|||
setattr(self, key, value)
|
||||
|
||||
|
||||
def text_tokens_without_nested_reasoning(
|
||||
completion_tokens: int,
|
||||
text_tokens: int,
|
||||
reasoning_tokens: int,
|
||||
other_modality_tokens: int,
|
||||
) -> int:
|
||||
reported_total: Final = text_tokens + reasoning_tokens + other_modality_tokens
|
||||
nested_reasoning_tokens: Final = min(reasoning_tokens, text_tokens, max(reported_total - completion_tokens, 0))
|
||||
return text_tokens - nested_reasoning_tokens
|
||||
|
||||
|
||||
class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for older openai versions
|
||||
text_tokens: int | None = None
|
||||
"""Text tokens generated by the model."""
|
||||
|
|
|
|||
|
|
@ -440,6 +440,56 @@ class TestResponseAPILoggingUtils:
|
|||
assert result.completion_tokens_details.text_tokens == 20
|
||||
assert result.completion_tokens_details.audio_tokens is None
|
||||
|
||||
def test_transform_realtime_usage_partitions_reasoning_out_of_text_tokens(self):
|
||||
"""Realtime nests reasoning_tokens inside text_tokens; the stored text share excludes them."""
|
||||
usage = {
|
||||
"input_tokens": 237,
|
||||
"output_tokens": 70,
|
||||
"total_tokens": 307,
|
||||
"input_token_details": {"text_tokens": 43, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
|
||||
"output_token_details": {"text_tokens": 70, "audio_tokens": 0, "reasoning_tokens": 52},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
|
||||
assert result.completion_tokens == 70
|
||||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.text_tokens == 18
|
||||
assert result.completion_tokens_details.reasoning_tokens == 52
|
||||
assert result.completion_tokens_details.audio_tokens == 0
|
||||
|
||||
def test_transform_realtime_usage_partitions_reasoning_beside_audio_output(self):
|
||||
"""Audio output stays as reported; only the text share sheds the nested reasoning tokens."""
|
||||
usage = {
|
||||
"input_tokens": 100,
|
||||
"output_tokens": 70,
|
||||
"total_tokens": 170,
|
||||
"input_token_details": {"text_tokens": 100, "audio_tokens": 0, "cached_tokens": 0},
|
||||
"output_token_details": {"text_tokens": 39, "audio_tokens": 31, "reasoning_tokens": 23},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
|
||||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.text_tokens == 16
|
||||
assert result.completion_tokens_details.audio_tokens == 31
|
||||
assert result.completion_tokens_details.reasoning_tokens == 23
|
||||
|
||||
def test_transform_response_api_usage_keeps_partitioned_text_tokens(self):
|
||||
"""A provider already reporting text_tokens beside reasoning_tokens is stored as sent."""
|
||||
usage = {
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 20,
|
||||
"total_tokens": 30,
|
||||
"output_tokens_details": {"text_tokens": 12, "reasoning_tokens": 5},
|
||||
}
|
||||
|
||||
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
|
||||
|
||||
assert result.completion_tokens_details is not None
|
||||
assert result.completion_tokens_details.text_tokens == 12
|
||||
assert result.completion_tokens_details.reasoning_tokens == 5
|
||||
|
||||
def test_transform_response_api_usage_carries_extra_provider_fields(self):
|
||||
"""Non-standard usage fields (e.g. xAI tool details) must survive chat normalization."""
|
||||
details = {"web_search_calls": 2, "x_search_calls": 0}
|
||||
|
|
|
|||
|
|
@ -4534,3 +4534,42 @@ def test_handle_realtime_stream_cost_calculation_bills_nested_reasoning_tokens_o
|
|||
)
|
||||
assert total_cost == pytest.approx(expected)
|
||||
assert total_cost == pytest.approx(0.0002362)
|
||||
|
||||
|
||||
def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() -> None:
|
||||
"""The combined usage that lands in spend logs keeps reasoning out of text_tokens for every turn."""
|
||||
results: OpenAIRealtimeStreamList = [
|
||||
{"type": "session.created", "session": {"model": "gpt-realtime-2.1-mini"}},
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {
|
||||
"total_tokens": 307,
|
||||
"input_tokens": 237,
|
||||
"output_tokens": 70,
|
||||
"input_token_details": {"text_tokens": 43, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
|
||||
"output_token_details": {"text_tokens": 70, "audio_tokens": 0, "reasoning_tokens": 52},
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "response.done",
|
||||
"response": {
|
||||
"usage": {
|
||||
"total_tokens": 363,
|
||||
"input_tokens": 300,
|
||||
"output_tokens": 63,
|
||||
"input_token_details": {"text_tokens": 106, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
|
||||
"output_token_details": {"text_tokens": 63, "audio_tokens": 0, "reasoning_tokens": 43},
|
||||
}
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results)
|
||||
|
||||
assert combined.completion_tokens == 133
|
||||
assert combined.completion_tokens_details is not None
|
||||
assert combined.completion_tokens_details.reasoning_tokens == 95
|
||||
assert combined.completion_tokens_details.text_tokens == 38
|
||||
assert combined.completion_tokens_details.audio_tokens == 0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue