mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-09 08:18:12 +00:00
fix(responses): report unreported cache writes as 0 so SDK 3 can parse usage
SDK 3's Responses usage schema requires input_tokens_details.cache_write_tokens, so a strict parse of a provider answer that reported no cache writes failed. The 0 is added only when the usage is serialized, so the cost path still treats an unreported value as unknown and a provider's cache_creation_tokens keeps pricing
This commit is contained in:
parent
3d8b7ffbdd
commit
6b7ccf247a
5 changed files with 50 additions and 5 deletions
|
|
@ -84,7 +84,10 @@ def _prompt_tokens_details_wrapper(details: object) -> PromptTokensDetailsWrappe
|
|||
if _is_object_dict(details):
|
||||
input_details: Final = InputTokensDetails.model_validate(details)
|
||||
return PromptTokensDetailsWrapper(
|
||||
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
|
||||
**{
|
||||
**input_details.model_dump(exclude={"cache_write_tokens_details"}),
|
||||
"cache_write_tokens": input_details.cache_write_tokens,
|
||||
},
|
||||
cache_creation_token_details=_cache_write_lifetimes(input_details.cache_write_tokens_details),
|
||||
)
|
||||
return PromptTokensDetailsWrapper(
|
||||
|
|
|
|||
|
|
@ -1414,7 +1414,7 @@ class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
audio_tokens: int | None = None
|
||||
cached_tokens: int = 0
|
||||
cached_tokens_details: CachedTokensDetails | None = None
|
||||
cache_write_tokens: int | None = Field(default=None, exclude_if=_is_none)
|
||||
cache_write_tokens: int | None = None
|
||||
cache_write_tokens_details: CacheWriteTokensDetails | None = Field(default=None, exclude_if=_is_none)
|
||||
image_tokens: int | None = None
|
||||
text_tokens: int | None = None
|
||||
|
|
@ -1422,6 +1422,13 @@ class InputTokensDetails(BaseLiteLLMOpenAIResponseObject):
|
|||
|
||||
model_config = {"extra": "allow"}
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _report_unreported_cache_writes_as_zero( # noqa: ANN202 # annotating it replaces the model's serialization schema
|
||||
self, handler: SerializerFunctionWrapHandler
|
||||
):
|
||||
serialized: Final[Mapping[str, object]] = handler(self)
|
||||
return {**serialized, "cache_write_tokens": self.cache_write_tokens or 0}
|
||||
|
||||
|
||||
class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
|
||||
input_tokens: int
|
||||
|
|
|
|||
|
|
@ -3439,9 +3439,9 @@ class TestUsageTransformation:
|
|||
assert response_usage.input_tokens_details is not None
|
||||
assert response_usage.input_tokens_details.cached_tokens == 5
|
||||
assert response_usage.input_tokens_details.text_tokens == 8
|
||||
assert {"cache_write_tokens", "cache_write_tokens_details"}.isdisjoint(
|
||||
response_usage.input_tokens_details.model_dump()
|
||||
)
|
||||
dumped_details: Final = response_usage.input_tokens_details.model_dump()
|
||||
assert dumped_details["cache_write_tokens"] == 0
|
||||
assert "cache_write_tokens_details" not in dumped_details
|
||||
|
||||
def test_transform_usage_with_cached_tokens_gemini(self):
|
||||
"""Test that cached_tokens from Gemini are properly transformed to input_tokens_details"""
|
||||
|
|
|
|||
|
|
@ -415,6 +415,20 @@ class TestResponseAPILoggingUtils:
|
|||
assert result.prompt_tokens_details.cache_creation_tokens == 10059
|
||||
assert result.prompt_tokens_details.cached_tokens == 0
|
||||
|
||||
def test_transform_response_api_usage_keeps_cache_creation_tokens_without_cache_writes(self) -> None:
|
||||
unvalidated_usage: Final = ResponseAPIUsage.model_construct(
|
||||
input_tokens=100,
|
||||
output_tokens=1,
|
||||
total_tokens=101,
|
||||
input_tokens_details={"cached_tokens": 0, "cache_creation_tokens": 50},
|
||||
)
|
||||
|
||||
usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(unvalidated_usage)
|
||||
|
||||
assert usage.prompt_tokens_details is not None
|
||||
assert usage.prompt_tokens_details.cache_creation_tokens == 50
|
||||
assert usage.prompt_tokens_details.cache_write_tokens == 50
|
||||
|
||||
@pytest.mark.parametrize("as_model", [False, True])
|
||||
def test_transform_response_api_usage_prices_12h_cache_writes(self, as_model: bool) -> None:
|
||||
usage_data: Final = {
|
||||
|
|
|
|||
|
|
@ -8,14 +8,19 @@ import pytest
|
|||
|
||||
import json
|
||||
|
||||
from openai.types.responses.response_usage import ResponseUsage as OpenAIResponseUsage
|
||||
|
||||
import litellm
|
||||
from litellm.types.llms.openai import (
|
||||
ContentPartDoneEvent,
|
||||
ContentPartDonePartOutputText,
|
||||
HttpxBinaryResponseContent,
|
||||
InputTokensDetails,
|
||||
OpenAIChatCompletionLogprobsContent,
|
||||
OpenAIModerationResponse,
|
||||
OpenAIVideoObject,
|
||||
OutputTokensDetails,
|
||||
ResponseAPIUsage,
|
||||
ResponseCompletedEvent,
|
||||
ResponsesAPIResponse,
|
||||
)
|
||||
|
|
@ -799,3 +804,19 @@ def test_rebuilding_a_dumped_responses_answer_keeps_the_tool_search_call_keys(ex
|
|||
rebuilt: Final = ResponsesAPIResponse(**answer.model_dump())
|
||||
|
||||
assert rebuilt.model_dump()["output"] == [search_call]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("cache_write_tokens", "reported"), ((None, 0), (7, 7)))
|
||||
def test_responses_usage_dump_matches_the_openai_sdk_usage_schema(cache_write_tokens: int | None, reported: int) -> None:
|
||||
usage: Final = ResponseAPIUsage(
|
||||
input_tokens=10,
|
||||
output_tokens=2,
|
||||
total_tokens=12,
|
||||
input_tokens_details=InputTokensDetails(cached_tokens=5, cache_write_tokens=cache_write_tokens),
|
||||
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
|
||||
)
|
||||
|
||||
sdk_usage: Final = OpenAIResponseUsage.model_validate(usage.model_dump(exclude_none=True))
|
||||
|
||||
assert sdk_usage.input_tokens_details.cache_write_tokens == reported
|
||||
assert sdk_usage.input_tokens_details.cached_tokens == 5
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue