From df2ec624ac1332b011b1949929a3d3285e331d22 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:13:02 +0000 Subject: [PATCH] fix(responses_adapters): map OpenAI Responses cache tokens on /v1/messages --- .../responses_adapters/streaming_iterator.py | 39 ++++- .../responses_adapters/transformation.py | 7 + ...t_responses_adapters_streaming_iterator.py | 133 ++++++++++++++++++ .../test_responses_adapters_transformation.py | 32 +++++ 4 files changed, 206 insertions(+), 5 deletions(-) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 4fd49a35417..328c731847e 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -9,6 +9,39 @@ from litellm import verbose_logger from litellm._uuid import uuid +def _coerce_int(value: object) -> int: + return value if isinstance(value, int) else 0 + + +def _extract_cache_tokens(usage: object) -> tuple[int, int]: + """Return (cache_read_tokens, cache_creation_tokens) from a Responses usage object. + + Anthropic-native names (cache_read_input_tokens / cache_creation_input_tokens) win when + present. Otherwise fall back to the OpenAI Responses shape, where the split lives under + input_tokens_details as cached_tokens and cache_write_tokens (a pydantic extra, so it is + read via model_dump() rather than a fixed getattr list). + """ + cache_read = _coerce_int(getattr(usage, "cache_read_input_tokens", 0)) + cache_creation = _coerce_int(getattr(usage, "cache_creation_input_tokens", 0)) + if cache_read and cache_creation: + return cache_read, cache_creation + + details = getattr(usage, "input_tokens_details", None) + if isinstance(details, dict): + details_dict = details + else: + dump = getattr(details, "model_dump", None) + details_dict = dump() if callable(dump) else {} + + if not cache_read: + cache_read = _coerce_int(details_dict.get("cached_tokens")) + if not cache_creation: + cache_creation = _coerce_int( + details_dict.get("cache_write_tokens") or details_dict.get("cache_creation_tokens") + ) + return cache_read, cache_creation + + class AnthropicResponsesStreamWrapper: """ Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. @@ -239,11 +272,7 @@ class AnthropicResponsesStreamWrapper: if usage is not None: input_tokens = getattr(usage, "input_tokens", 0) or 0 output_tokens = getattr(usage, "output_tokens", 0) or 0 - cache_creation_tokens = getattr(usage, "input_tokens_details", None) # type: ignore[assignment] - cache_read_tokens = getattr(usage, "output_tokens_details", None) # type: ignore[assignment] - # Prefer direct cache fields if present - cache_creation_tokens = int(getattr(usage, "cache_creation_input_tokens", 0) or 0) - cache_read_tokens = int(getattr(usage, "cache_read_input_tokens", 0) or 0) + cache_read_tokens, cache_creation_tokens = _extract_cache_tokens(usage) # Check if tool_use was in the output to override stop_reason if response_obj is not None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 172e54de98e..265af2c0b83 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -31,6 +31,8 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( ) from litellm.types.llms.openai import ResponsesAPIResponse +from .streaming_iterator import _extract_cache_tokens + class LiteLLMAnthropicToResponsesAPIAdapter: """ @@ -467,11 +469,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter: raw_usage: Optional[ResponseAPIUsage] = response.usage input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + cache_read_tokens, cache_creation_tokens = _extract_cache_tokens(raw_usage) anthropic_usage = AnthropicUsage( input_tokens=input_tokens, output_tokens=output_tokens, ) + if cache_read_tokens: + anthropic_usage["cache_read_input_tokens"] = cache_read_tokens + if cache_creation_tokens: + anthropic_usage["cache_creation_input_tokens"] = cache_creation_tokens return AnthropicMessagesResponse( id=response.id, diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 9b5197d9028..b414b4c84bc 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -7,6 +7,8 @@ import asyncio import os import sys +from pydantic import BaseModel, ConfigDict + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../../.."))) from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streaming_iterator import ( @@ -14,6 +16,35 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.streami ) +class _Details(BaseModel): + model_config = ConfigDict(extra="allow") + cached_tokens: int = 0 + + +class _Usage(BaseModel): + model_config = ConfigDict(extra="allow") + input_tokens: int + output_tokens: int + input_tokens_details: _Details + + +class _Response(BaseModel): + model_config = ConfigDict(extra="allow") + status: str = "completed" + usage: _Usage + output: list = [] + + +class _Event(BaseModel): + model_config = ConfigDict(extra="allow") + type: str = "response.completed" + response: _Response + + +def _completed_usage(events: list) -> dict: + return next(c for c in _process_all(events) if c.get("type") == "message_delta")["usage"] + + def _process_all(events: list) -> list: wrapper = AnthropicResponsesStreamWrapper(responses_stream=None, model="m") for event in events: @@ -130,3 +161,105 @@ class TestProcessEventTextDeltaWithoutOutputItemAdded: ("content_block_start", 0), ("content_block_delta", 0), ] + + +class TestResponseCompletedCacheTokens: + """message_delta must surface OpenAI Responses cache-read/write counts. OpenAI reports + them under input_tokens_details (cached_tokens / cache_write_tokens), not as the + Anthropic-native cache_*_input_tokens names the adapter previously read, so cache reads + were always billed at the full input rate. See issue #35127.""" + + def test_openai_cached_tokens_from_model_details(self): + event = _Event( + response=_Response( + usage=_Usage( + input_tokens=12000, + output_tokens=40, + input_tokens_details=_Details(cached_tokens=11008), + ) + ) + ) + assert _completed_usage([event]) == { + "input_tokens": 12000, + "output_tokens": 40, + "cache_read_input_tokens": 11008, + } + + def test_openai_cached_tokens_from_dict_details(self): + class _DictDetailsUsage(BaseModel): + model_config = ConfigDict(extra="allow") + input_tokens: int + output_tokens: int + input_tokens_details: dict + + event = _Event.model_construct( + response=_Response.model_construct( + status="completed", + output=[], + usage=_DictDetailsUsage( + input_tokens=12000, + output_tokens=40, + input_tokens_details={"cached_tokens": 11008}, + ), + ) + ) + assert _completed_usage([event]) == { + "input_tokens": 12000, + "output_tokens": 40, + "cache_read_input_tokens": 11008, + } + + def test_cache_write_tokens_pydantic_extra_becomes_cache_creation(self): + event = _Event( + response=_Response( + usage=_Usage( + input_tokens=172000, + output_tokens=40, + input_tokens_details=_Details(cached_tokens=155000, cache_write_tokens=3000), + ) + ) + ) + assert _completed_usage([event]) == { + "input_tokens": 172000, + "output_tokens": 40, + "cache_read_input_tokens": 155000, + "cache_creation_input_tokens": 3000, + } + + def test_anthropic_native_fields_take_precedence(self): + class _AnthropicUsage(BaseModel): + model_config = ConfigDict(extra="allow") + input_tokens: int + output_tokens: int + cache_read_input_tokens: int + cache_creation_input_tokens: int + input_tokens_details: _Details + + event = _Event.model_construct( + response=_Response.model_construct( + status="completed", + output=[], + usage=_AnthropicUsage( + input_tokens=500, + output_tokens=40, + cache_read_input_tokens=100, + cache_creation_input_tokens=25, + input_tokens_details=_Details(cached_tokens=999), + ), + ) + ) + usage = _completed_usage([event]) + assert usage["cache_read_input_tokens"] == 100 + assert usage["cache_creation_input_tokens"] == 25 + + def test_absent_cache_fields_omit_keys(self): + event = _Event( + response=_Response( + usage=_Usage( + input_tokens=500, + output_tokens=40, + input_tokens_details=_Details(cached_tokens=0), + ) + ) + ) + assert _completed_usage([event]) == {"input_tokens": 500, "output_tokens": 40} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 606ff39b35e..0f4a8500b91 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -961,6 +961,38 @@ class TestTranslateResponse: assert result["usage"]["input_tokens"] == 200 assert result["usage"]["output_tokens"] == 75 + def test_openai_cache_tokens_mapped_to_anthropic_usage(self): + """OpenAI Responses reports cache reads/writes under input_tokens_details; they must + surface as Anthropic cache_read_input_tokens / cache_creation_input_tokens so cache + reads are not billed at the full input rate. See issue #35127.""" + from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage + + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage = ResponseAPIUsage( + input_tokens=172000, + output_tokens=40, + total_tokens=172040, + input_tokens_details=InputTokensDetails(cached_tokens=155000, cache_write_tokens=3000), + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["cache_read_input_tokens"] == 155000 + assert result["usage"]["cache_creation_input_tokens"] == 3000 + + def test_no_cache_tokens_omits_keys(self): + """Responses without a cache split must not emit zero-valued cache keys.""" + from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage + + response = _make_mock_response(output=[_make_output_message(["OK"])]) + response.usage = ResponseAPIUsage( + input_tokens=500, + output_tokens=40, + total_tokens=540, + input_tokens_details=InputTokensDetails(cached_tokens=0), + ) + result: Any = _ADAPTER.translate_response(response) + assert "cache_read_input_tokens" not in result["usage"] + assert "cache_creation_input_tokens" not in result["usage"] + def test_model_and_id_preserved(self): """Model and response ID from the Responses API are forwarded.""" response = _make_mock_response(