fix(realtime): store text_tokens without the nested reasoning share

The realtime usage writer passed the provider's output_token_details through as sent, so spend logs and callbacks kept a text_tokens that still contained reasoning_tokens while every other completion_tokens_details producer stores the partitioned share. The writer now applies the same rule the cost calculator uses, moved to litellm/types/utils.py so both read one definition, and the calculator keeps it for usage objects that arrive nested from elsewhere
This commit is contained in:
mateo-berri 2026-09-04 22:47:09 -07:00
parent 6294367920
commit e1d900d1c2
5 changed files with 123 additions and 16 deletions

View file

@ -26,6 +26,7 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
ServiceTier,
Usage,
text_tokens_without_nested_reasoning,
)
from litellm.utils import get_model_info
@ -852,17 +853,6 @@ class CompletionTokensDetailsResult(TypedDict):
video_tokens: int
def _text_tokens_without_nested_reasoning(
completion_tokens: int,
text_tokens: int,
reasoning_tokens: int,
other_modality_tokens: int,
) -> int:
reported_total: Final = text_tokens + reasoning_tokens + other_modality_tokens
nested_reasoning_tokens: Final = min(reasoning_tokens, text_tokens, max(reported_total - completion_tokens, 0))
return text_tokens - nested_reasoning_tokens
def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
audio_tokens: Final = (
cast(
@ -893,7 +883,7 @@ def parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResu
or 0
)
video_tokens: Final = _coerce_token_count(getattr(usage.completion_tokens_details, "video_tokens", 0))
text_tokens: Final = _text_tokens_without_nested_reasoning(
text_tokens: Final = text_tokens_without_nested_reasoning(
completion_tokens=usage.completion_tokens,
text_tokens=reported_text_tokens,
reasoning_tokens=reasoning_tokens,

View file

@ -25,9 +25,15 @@ from litellm.types.utils import (
PromptTokensDetailsWrapper,
SpecialEnums,
Usage,
text_tokens_without_nested_reasoning,
)
def _output_token_detail(details: object, field: str) -> int | None:
value: Final = getattr(details, field, None)
return value if isinstance(value, int) else None
def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]: # guard-ok: a list is a Sequence of anything
return isinstance(value, list)
@ -1134,11 +1140,22 @@ class ResponseAPILoggingUtils:
response_api_usage, "output_tokens_details", None
)
if output_tokens_details:
reasoning_tokens: Final = _output_token_detail(output_tokens_details, "reasoning_tokens")
image_tokens: Final = _output_token_detail(output_tokens_details, "image_tokens")
audio_tokens: Final = _output_token_detail(output_tokens_details, "audio_tokens")
reported_text_tokens: Final = _output_token_detail(output_tokens_details, "text_tokens")
completion_tokens_details = CompletionTokensDetailsWrapper(
reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None),
image_tokens=getattr(output_tokens_details, "image_tokens", None),
text_tokens=getattr(output_tokens_details, "text_tokens", None),
audio_tokens=getattr(output_tokens_details, "audio_tokens", None),
reasoning_tokens=reasoning_tokens,
image_tokens=image_tokens,
text_tokens=None
if reported_text_tokens is None
else text_tokens_without_nested_reasoning(
completion_tokens=completion_tokens,
text_tokens=reported_text_tokens,
reasoning_tokens=reasoning_tokens or 0,
other_modality_tokens=(audio_tokens or 0) + (image_tokens or 0),
),
audio_tokens=audio_tokens,
)
extra_usage_fields: Final = {

View file

@ -1622,6 +1622,17 @@ class Choices(SafeAttributeModel, OpenAIObject):
setattr(self, key, value)
def text_tokens_without_nested_reasoning(
completion_tokens: int,
text_tokens: int,
reasoning_tokens: int,
other_modality_tokens: int,
) -> int:
reported_total: Final = text_tokens + reasoning_tokens + other_modality_tokens
nested_reasoning_tokens: Final = min(reasoning_tokens, text_tokens, max(reported_total - completion_tokens, 0))
return text_tokens - nested_reasoning_tokens
class CompletionTokensDetailsWrapper(CompletionTokensDetails): # wrapper for older openai versions
text_tokens: int | None = None
"""Text tokens generated by the model."""

View file

@ -440,6 +440,56 @@ class TestResponseAPILoggingUtils:
assert result.completion_tokens_details.text_tokens == 20
assert result.completion_tokens_details.audio_tokens is None
def test_transform_realtime_usage_partitions_reasoning_out_of_text_tokens(self):
"""Realtime nests reasoning_tokens inside text_tokens; the stored text share excludes them."""
usage = {
"input_tokens": 237,
"output_tokens": 70,
"total_tokens": 307,
"input_token_details": {"text_tokens": 43, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
"output_token_details": {"text_tokens": 70, "audio_tokens": 0, "reasoning_tokens": 52},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.completion_tokens == 70
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.text_tokens == 18
assert result.completion_tokens_details.reasoning_tokens == 52
assert result.completion_tokens_details.audio_tokens == 0
def test_transform_realtime_usage_partitions_reasoning_beside_audio_output(self):
"""Audio output stays as reported; only the text share sheds the nested reasoning tokens."""
usage = {
"input_tokens": 100,
"output_tokens": 70,
"total_tokens": 170,
"input_token_details": {"text_tokens": 100, "audio_tokens": 0, "cached_tokens": 0},
"output_token_details": {"text_tokens": 39, "audio_tokens": 31, "reasoning_tokens": 23},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.text_tokens == 16
assert result.completion_tokens_details.audio_tokens == 31
assert result.completion_tokens_details.reasoning_tokens == 23
def test_transform_response_api_usage_keeps_partitioned_text_tokens(self):
"""A provider already reporting text_tokens beside reasoning_tokens is stored as sent."""
usage = {
"input_tokens": 10,
"output_tokens": 20,
"total_tokens": 30,
"output_tokens_details": {"text_tokens": 12, "reasoning_tokens": 5},
}
result = ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(usage)
assert result.completion_tokens_details is not None
assert result.completion_tokens_details.text_tokens == 12
assert result.completion_tokens_details.reasoning_tokens == 5
def test_transform_response_api_usage_carries_extra_provider_fields(self):
"""Non-standard usage fields (e.g. xAI tool details) must survive chat normalization."""
details = {"web_search_calls": 2, "x_search_calls": 0}

View file

@ -4534,3 +4534,42 @@ def test_handle_realtime_stream_cost_calculation_bills_nested_reasoning_tokens_o
)
assert total_cost == pytest.approx(expected)
assert total_cost == pytest.approx(0.0002362)
def test_collect_and_combine_realtime_usage_stores_partitioned_text_tokens() -> None:
"""The combined usage that lands in spend logs keeps reasoning out of text_tokens for every turn."""
results: OpenAIRealtimeStreamList = [
{"type": "session.created", "session": {"model": "gpt-realtime-2.1-mini"}},
{
"type": "response.done",
"response": {
"usage": {
"total_tokens": 307,
"input_tokens": 237,
"output_tokens": 70,
"input_token_details": {"text_tokens": 43, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
"output_token_details": {"text_tokens": 70, "audio_tokens": 0, "reasoning_tokens": 52},
}
},
},
{
"type": "response.done",
"response": {
"usage": {
"total_tokens": 363,
"input_tokens": 300,
"output_tokens": 63,
"input_token_details": {"text_tokens": 106, "audio_tokens": 0, "image_tokens": 194, "cached_tokens": 0},
"output_token_details": {"text_tokens": 63, "audio_tokens": 0, "reasoning_tokens": 43},
}
},
},
]
combined = RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=results)
assert combined.completion_tokens == 133
assert combined.completion_tokens_details is not None
assert combined.completion_tokens_details.reasoning_tokens == 95
assert combined.completion_tokens_details.text_tokens == 38
assert combined.completion_tokens_details.audio_tokens == 0