From e0300384cdd1762d272852167dcdbfa40d4b179e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:33:40 +0000 Subject: [PATCH] fix(anthropic): map Responses API cached input tokens to cache_read_input_tokens The Anthropic /v1/messages bridge for Responses API models never populated cache_read_input_tokens, so clients reading the Anthropic shape saw an uncached prompt on every call even when upstream served ~100% of it from cache. Streaming read the cache counts from Anthropic-named attributes that a Responses usage object does not have, and the non-streaming path dropped them entirely. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../responses_adapters/streaming_iterator.py | 32 ++++--------- .../responses_adapters/transformation.py | 16 ++----- .../responses_adapters/usage.py | 46 +++++++++++++++++++ ...t_responses_adapters_streaming_iterator.py | 39 ++++++++++++++++ .../test_responses_adapters_transformation.py | 32 +++++++++++-- 5 files changed, 128 insertions(+), 37 deletions(-) create mode 100644 litellm/llms/anthropic/experimental_pass_through/responses_adapters/usage.py diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py index 56d8a34ad5c..5730ebc720c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -8,6 +8,9 @@ from typing import Any, Final from litellm import verbose_logger from litellm._uuid import uuid +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.usage import ( + anthropic_usage_from_responses_usage, +) class AnthropicResponsesStreamWrapper: @@ -226,25 +229,19 @@ class AnthropicResponsesStreamWrapper: response_obj: Final = getattr(event, "response", None) or ( event.get("response") if isinstance(event, dict) else None ) + usage: Final = ( + getattr(response_obj, "usage", None) + or (response_obj.get("usage") if isinstance(response_obj, dict) else None) + if response_obj is not None + else None + ) + usage_delta: Final = anthropic_usage_from_responses_usage(usage) stop_reason = "end_turn" - input_tokens = 0 - output_tokens = 0 - cache_creation_tokens = 0 - cache_read_tokens = 0 if response_obj is not None: status: Final = getattr(response_obj, "status", None) if status == "incomplete": stop_reason = "max_tokens" - usage: Final = getattr(response_obj, "usage", None) - if usage is not None: - input_tokens = getattr(usage, "input_tokens", 0) or 0 - output_tokens = getattr(usage, "output_tokens", 0) or 0 - cache_creation_tokens = getattr(usage, "input_tokens_details", None) # type: ignore[assignment] - cache_read_tokens = getattr(usage, "output_tokens_details", None) # type: ignore[assignment] - # Prefer direct cache fields if present - cache_creation_tokens = int(getattr(usage, "cache_creation_input_tokens", 0) or 0) - cache_read_tokens = int(getattr(usage, "cache_read_input_tokens", 0) or 0) # Check if tool_use was in the output to override stop_reason if response_obj is not None: @@ -257,15 +254,6 @@ class AnthropicResponsesStreamWrapper: stop_reason = "tool_use" break - usage_delta: Final[dict[str, Any]] = { - "input_tokens": input_tokens, - "output_tokens": output_tokens, - } - if cache_creation_tokens: - usage_delta["cache_creation_input_tokens"] = cache_creation_tokens - if cache_read_tokens: - usage_delta["cache_read_input_tokens"] = cache_read_tokens - self._chunk_queue.append( { "type": "message_delta", diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 1fb5a88cb2b..bdde9594cdb 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -11,6 +11,9 @@ from typing import Any, Final, cast from litellm.litellm_core_utils.reasoning_effort_utils import ( reasoning_effort_from_thinking_budget, ) +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.usage import ( + anthropic_usage_from_responses_usage, +) from litellm.llms.anthropic.experimental_pass_through.utils import ( is_reasoning_auto_summary_enabled, ) @@ -27,7 +30,6 @@ from litellm.types.llms.anthropic import ( ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, - AnthropicUsage, ) from litellm.types.llms.openai import ResponsesAPIResponse @@ -386,8 +388,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter: ResponseReasoningItem, ) - from litellm.types.llms.openai import ResponseAPIUsage - content: Final[list[dict[str, Any]]] = [] stop_reason: AnthropicFinishReason = "end_turn" @@ -453,15 +453,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter: if response.status == "incomplete": stop_reason = "max_tokens" - # usage - raw_usage: Final[ResponseAPIUsage | None] = response.usage - input_tokens: Final = int(getattr(raw_usage, "input_tokens", 0) or 0) - output_tokens: Final = int(getattr(raw_usage, "output_tokens", 0) or 0) - - anthropic_usage: Final = AnthropicUsage( - input_tokens=input_tokens, - output_tokens=output_tokens, - ) + anthropic_usage: Final = anthropic_usage_from_responses_usage(response.usage) return AnthropicMessagesResponse( id=response.id, diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/usage.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/usage.py new file mode 100644 index 00000000000..01de8cb99bc --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/usage.py @@ -0,0 +1,46 @@ +from typing import Final + +from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicUsage +from litellm.types.llms.openai import ResponseAPIUsage + + +def _positive_int(value: object) -> int: + if isinstance(value, bool): + return 0 + if isinstance(value, int) and value > 0: + return value + return 0 + + +def _field(usage: ResponseAPIUsage | dict[str, object] | None, field_name: str) -> object: + if isinstance(usage, dict): + return usage.get(field_name) + return getattr(usage, field_name, None) + + +def _cached_input_tokens(usage: ResponseAPIUsage | dict[str, object] | None) -> int: + details: Final = _field(usage, "input_tokens_details") + if details is None: + return 0 + if isinstance(details, dict): + return _positive_int(details.get("cached_tokens")) + return _positive_int(getattr(details, "cached_tokens", None)) + + +def anthropic_usage_from_responses_usage(usage: ResponseAPIUsage | dict[str, object] | None) -> AnthropicUsage: + """ + The Responses API counts cache hits inside ``input_tokens`` and reports them in + ``input_tokens_details.cached_tokens``, while Anthropic clients read + ``cache_read_input_tokens`` and expect ``input_tokens`` to exclude it. + """ + input_tokens: Final = _positive_int(_field(usage, "input_tokens")) + output_tokens: Final = _positive_int(_field(usage, "output_tokens")) + cache_read_input_tokens: Final = min(_cached_input_tokens(usage), input_tokens) + + if cache_read_input_tokens > 0: + return AnthropicUsage( + input_tokens=input_tokens - cache_read_input_tokens, + output_tokens=output_tokens, + cache_read_input_tokens=cache_read_input_tokens, + ) + return AnthropicUsage(input_tokens=input_tokens, output_tokens=output_tokens) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py index 9b5197d9028..b85be589edb 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_streaming_iterator.py @@ -130,3 +130,42 @@ class TestProcessEventTextDeltaWithoutOutputItemAdded: ("content_block_start", 0), ("content_block_delta", 0), ] + + +class TestResponseCompletedUsage: + """The Responses API reports cache hits in ``input_tokens_details.cached_tokens`` + and counts them inside ``input_tokens``, while Anthropic clients read + ``cache_read_input_tokens`` from the ``message_delta`` usage and expect + ``input_tokens`` to exclude it.""" + + @staticmethod + def _message_delta_usage(usage: dict) -> dict: + chunks = _process_all([{"type": "response.completed", "response": {"status": "completed", "usage": usage}}]) + return next(chunk["usage"] for chunk in chunks if chunk["type"] == "message_delta") + + def test_cached_tokens_mapped_to_cache_read_input_tokens(self): + usage = self._message_delta_usage( + { + "input_tokens": 25616, + "input_tokens_details": {"cached_tokens": 25613}, + "output_tokens": 5, + } + ) + assert usage == { + "input_tokens": 3, + "output_tokens": 5, + "cache_read_input_tokens": 25613, + } + + def test_no_cache_hit_omits_cache_fields(self): + usage = self._message_delta_usage( + { + "input_tokens": 120, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 7, + } + ) + assert usage == {"input_tokens": 120, "output_tokens": 7} + + def test_missing_usage_yields_zeroes(self): + assert self._message_delta_usage({}) == {"input_tokens": 0, "output_tokens": 0} diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py index 606ff39b35e..d4c45d17ef2 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -20,6 +20,7 @@ from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transfo LiteLLMAnthropicToResponsesAPIAdapter, ) from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.openai import InputTokensDetails, ResponseAPIUsage def _make_request(**overrides) -> AnthropicMessagesRequest: @@ -823,11 +824,15 @@ def _make_mock_response( model: str = "gpt-4o", input_tokens: int = 100, output_tokens: int = 50, + cached_tokens: int = 0, ) -> MagicMock: """Build a minimal mock ResponsesAPIResponse.""" - usage = MagicMock() - usage.input_tokens = input_tokens - usage.output_tokens = output_tokens + usage = ResponseAPIUsage( + input_tokens=input_tokens, + input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens), + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + ) resp = MagicMock() resp.id = response_id @@ -961,6 +966,27 @@ class TestTranslateResponse: assert result["usage"]["input_tokens"] == 200 assert result["usage"]["output_tokens"] == 75 + def test_cached_tokens_mapped_to_cache_read_input_tokens(self): + """The Responses API reports cache hits in ``input_tokens_details.cached_tokens`` + and counts them inside ``input_tokens``, while Anthropic clients read + ``cache_read_input_tokens`` and expect ``input_tokens`` to exclude it.""" + response = _make_mock_response( + output=[_make_output_message(["OK"])], + input_tokens=25616, + output_tokens=5, + cached_tokens=25613, + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["cache_read_input_tokens"] == 25613 + assert result["usage"]["input_tokens"] == 3 + assert result["usage"]["output_tokens"] == 5 + + def test_no_cache_hit_omits_cache_fields(self): + response = _make_mock_response(output=[_make_output_message(["OK"])], input_tokens=200, output_tokens=75) + result: Any = _ADAPTER.translate_response(response) + assert "cache_read_input_tokens" not in result["usage"] + assert "cache_creation_input_tokens" not in result["usage"] + def test_model_and_id_preserved(self): """Model and response ID from the Responses API are forwarded.""" response = _make_mock_response(