From 20545eda8076f9a8936ef24f4caa0aa4228c30ae Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" Date: Sat, 8 Aug 2026 19:09:56 +0000 Subject: [PATCH] fix(anthropic): use provider-reported thinking_tokens for reasoning_tokens --- litellm/llms/anthropic/chat/transformation.py | 22 +++++- .../test_anthropic_chat_transformation.py | 79 +++++++++++++++++++ 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 1161c92232a..7d1d646da55 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2117,6 +2117,22 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return False return any(key in usage_object for key in ("cache_read_input_tokens", "cache_creation_input_tokens")) + @staticmethod + def _get_reported_thinking_tokens(usage_object: dict) -> int | None: + """Anthropic reports the billed extended-thinking count in ``usage.output_tokens_details.thinking_tokens``. + + It is authoritative: under the default ``display: "omitted"`` the thinking blocks carry no text at all, + so estimating from the visible reasoning content always yields 0, and under ``display: "summarized"`` + the visible summary understates what was billed. + """ + details: Final = usage_object.get("output_tokens_details") + if not isinstance(details, dict): + return None + thinking_tokens: Final = details.get("thinking_tokens") + if isinstance(thinking_tokens, bool) or not isinstance(thinking_tokens, (int, float)): + return None + return int(thinking_tokens) + def calculate_usage( self, usage_object: dict, @@ -2199,7 +2215,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): estimated_reasoning_tokens: Final = ( token_counter(text=reasoning_content, count_response_tokens=True) if reasoning_content else 0 ) - reasoning_tokens: Final = min(estimated_reasoning_tokens, completion_tokens) + reported_reasoning_tokens: Final = self._get_reported_thinking_tokens(_usage) + reasoning_tokens: Final = min( + estimated_reasoning_tokens if reported_reasoning_tokens is None else reported_reasoning_tokens, + completion_tokens, + ) completion_token_details: Final = CompletionTokensDetailsWrapper( reasoning_tokens=max(0, reasoning_tokens), text_tokens=(completion_tokens - reasoning_tokens if reasoning_tokens > 0 else completion_tokens), diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 231d3b48754..6fc61da326a 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -2436,6 +2436,85 @@ def test_calculate_usage_completion_tokens_details_with_reasoning(): assert usage.completion_tokens == 500 +def test_calculate_usage_prefers_reported_thinking_tokens(): + """ + Anthropic reports the billed thinking count in usage.output_tokens_details.thinking_tokens. + Under the default display ("omitted") the thinking blocks are empty, so estimating from the + visible reasoning text yields 0 even though the model thought (and billed). + + Fixes: https://github.com/BerriAI/litellm/issues/36290 + """ + config = AnthropicConfig() + + usage_object = { + "input_tokens": 29, + "output_tokens": 391, + "output_tokens_details": {"thinking_tokens": 180}, + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content="") + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 180 + assert usage.completion_tokens_details.text_tokens == 211 + + +def test_calculate_usage_reported_thinking_tokens_override_text_estimate(): + """Summarized thinking text understates the billed count, so the reported value wins.""" + config = AnthropicConfig() + + usage_object = { + "input_tokens": 29, + "output_tokens": 391, + "output_tokens_details": {"thinking_tokens": 180}, + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content="short summary") + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 180 + + +@pytest.mark.parametrize( + "output_tokens_details", + [None, {}, {"thinking_tokens": None}], +) +def test_calculate_usage_falls_back_to_estimate_without_reported_thinking_tokens(output_tokens_details): + """Older responses have no thinking_tokens, so the visible-text estimate must still apply.""" + config = AnthropicConfig() + + usage_object = { + "input_tokens": 100, + "output_tokens": 500, + "output_tokens_details": output_tokens_details, + } + + usage = config.calculate_usage( + usage_object=usage_object, + reasoning_content="Let me think about this step by step. " * 10, + ) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens is not None + assert usage.completion_tokens_details.reasoning_tokens > 0 + + +def test_calculate_usage_clamps_reported_thinking_tokens_to_output_tokens(): + config = AnthropicConfig() + + usage_object = { + "input_tokens": 10, + "output_tokens": 50, + "output_tokens_details": {"thinking_tokens": 900}, + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.reasoning_tokens == 50 + assert usage.completion_tokens_details.text_tokens == 0 + + # ============ Reasoning Effort Tests ============