diff --git a/litellm/llms/anthropic/messages/adapters/transformation.py b/litellm/llms/anthropic/messages/adapters/transformation.py index 5e4fba6d706..376769ef7c0 100644 --- a/litellm/llms/anthropic/messages/adapters/transformation.py +++ b/litellm/llms/anthropic/messages/adapters/transformation.py @@ -209,6 +209,7 @@ from litellm.types.llms.anthropic import ( ServerToolUsage, StreamingContentBlockDeltaType, UsageDelta, + UsageDeltaOutputTokensDetails, UsageIteration, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( @@ -1650,6 +1651,13 @@ class LiteLLMAnthropicMessagesAdapter: return from_server_tool_use return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",)) + @staticmethod + def _get_reasoning_tokens(usage: Usage) -> int: + details: Final = usage.completion_tokens_details + if details is None or details.reasoning_tokens is None: + return 0 + return details.reasoning_tokens + @classmethod def translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta: cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage) @@ -1668,12 +1676,19 @@ class LiteLLMAnthropicMessagesAdapter: usage_delta["cache_creation_input_tokens"] = cache_creation_input_tokens if cache_read_input_tokens > 0: usage_delta["cache_read_input_tokens"] = cache_read_input_tokens + reasoning_tokens: Final = cls._get_reasoning_tokens(usage) + usage_with_thinking: Final = ( + usage_delta + | UsageDelta(output_tokens_details=UsageDeltaOutputTokensDetails(thinking_tokens=reasoning_tokens)) + if reasoning_tokens > 0 + else usage_delta + ) if web_search_requests > 0: return UsageDelta( - **usage_delta, + **usage_with_thinking, server_tool_use=ServerToolUsage(web_search_requests=web_search_requests), ) - return usage_delta + return usage_with_thinking _translate_openai_usage_to_anthropic_usage_delta = translate_openai_usage_to_anthropic_usage_delta diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index c322eb25b3e..d8489bfb8f6 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -604,12 +604,17 @@ class ServerToolUsage(TypedDict, total=False): web_search_requests: ReadOnly[int] +class UsageDeltaOutputTokensDetails(TypedDict, total=False): + thinking_tokens: ReadOnly[int] + + class UsageDelta(TypedDict, total=False): input_tokens: int output_tokens: int cache_creation_input_tokens: int cache_read_input_tokens: int server_tool_use: ReadOnly[ServerToolUsage] + output_tokens_details: ReadOnly[UsageDeltaOutputTokensDetails] class AppliedEdit(TypedDict, total=False): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 51168071c85..7f6393f57c7 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -64,6 +64,10 @@ AnthropicResponseContentBlock: TypeAlias = ( ) +class AnthropicOutputTokensDetails(TypedDict, total=False): + thinking_tokens: ReadOnly[int] + + class AnthropicUsage(TypedDict, total=False): """ Input and output tokens used in the request @@ -84,6 +88,8 @@ class AnthropicUsage(TypedDict, total=False): """ server_tool_use: NotRequired[ReadOnly[ServerToolUsage]] + output_tokens_details: NotRequired[ReadOnly[AnthropicOutputTokensDetails]] + class AnthropicMessagesResponse(TypedDict, total=False): """ diff --git a/tests/integration/streaming/test_stream_contracts.py b/tests/integration/streaming/test_stream_contracts.py index 1d2a8b25792..488aa53f345 100644 --- a/tests/integration/streaming/test_stream_contracts.py +++ b/tests/integration/streaming/test_stream_contracts.py @@ -465,7 +465,11 @@ def test_messages_stream_opens_thinking_block_at_index_zero_for_reasoning_conten "message_stop", ), text message_delta: Final = next(event for event in events if event["type"] == "message_delta") - assert message_delta["usage"] == {"input_tokens": 11, "output_tokens": 6}, text + assert message_delta["usage"] == { + "input_tokens": 11, + "output_tokens": 6, + "output_tokens_details": {"thinking_tokens": 4}, + }, text requests: Final = wire.drain() assert len(requests) == 1 outbound: Final = json.loads(requests[0].body) diff --git a/tests/integration/translation/messages/bases/azure.py b/tests/integration/translation/messages/bases/azure.py index b731f7b7760..43e86daed7c 100644 --- a/tests/integration/translation/messages/bases/azure.py +++ b/tests/integration/translation/messages/bases/azure.py @@ -409,7 +409,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_0534b0c3898d6d1b006ad023ba700c8195b63f1819ab327ab3", "model": "azure/gpt-6-luna", - "usage": {"input_tokens": 309, "output_tokens": 114}, + "usage": {"input_tokens": 309, "output_tokens": 114, "output_tokens_details": {"thinking_tokens": 50}}, "content": [ { "type": "text", @@ -543,7 +543,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_0a4a896afe94ce65006ad023b047248193999d7c3a0c4fec39", "model": "azure/gpt-6.1-sol", - "usage": {"input_tokens": 309, "output_tokens": 137}, + "usage": {"input_tokens": 309, "output_tokens": 137, "output_tokens_details": {"thinking_tokens": 72}}, "content": [ { "type": "text", @@ -679,7 +679,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_019af305501e4822006ad023c3a50881958c4cf40623699cf8", "model": "azure/gpt-6-astra", - "usage": {"input_tokens": 309, "output_tokens": 138}, + "usage": {"input_tokens": 309, "output_tokens": 138, "output_tokens_details": {"thinking_tokens": 72}}, "content": [ { "type": "text", @@ -812,7 +812,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_0d178c5d2401a624006ad023cf1c588194ab41d5803ed29401", "model": "azure/gpt-5.6-terra", - "usage": {"input_tokens": 309, "output_tokens": 92}, + "usage": {"input_tokens": 309, "output_tokens": 92, "output_tokens_details": {"thinking_tokens": 22}}, "content": [ { "type": "text", diff --git a/tests/integration/translation/messages/bases/bedrock_chat_completions.py b/tests/integration/translation/messages/bases/bedrock_chat_completions.py index 9a59f40a09d..061693dc8cc 100644 --- a/tests/integration/translation/messages/bases/bedrock_chat_completions.py +++ b/tests/integration/translation/messages/bases/bedrock_chat_completions.py @@ -55,7 +55,7 @@ GROK_4_7_TEST_CASE: Final = TranslationTestCase( "role": "assistant", "model": "bedrock/global.xai.grok-4.7", "stop_sequence": None, - "usage": {"input_tokens": 30, "output_tokens": 37}, + "usage": {"input_tokens": 30, "output_tokens": 37, "output_tokens_details": {"thinking_tokens": 27}}, "content": [{"type": "text", "text": "Hello."}], "stop_reason": "end_turn", "stop_details": None, diff --git a/tests/integration/translation/messages/bases/bedrock_converse.py b/tests/integration/translation/messages/bases/bedrock_converse.py index 3d2b731d7c0..9870e768188 100644 --- a/tests/integration/translation/messages/bases/bedrock_converse.py +++ b/tests/integration/translation/messages/bases/bedrock_converse.py @@ -278,7 +278,12 @@ CLAUDE_HAIKU_5_5_TEST_CASE: Final = TranslationTestCase( **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, "model": "bedrock/converse/us.anthropic.claude-haiku-5-5", - "usage": {"input_tokens": 4, "output_tokens": 297, "cache_creation_input_tokens": 853}, + "usage": { + "input_tokens": 4, + "output_tokens": 297, + "cache_creation_input_tokens": 853, + "output_tokens_details": {"thinking_tokens": 147}, + }, "content": [ { "type": "thinking", @@ -404,7 +409,12 @@ CLAUDE_SONNET_5_5_TEST_CASE: Final = TranslationTestCase( **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, "model": "bedrock/converse/us.anthropic.claude-sonnet-5-5", - "usage": {"input_tokens": 4, "output_tokens": 430, "cache_creation_input_tokens": 853}, + "usage": { + "input_tokens": 4, + "output_tokens": 430, + "cache_creation_input_tokens": 853, + "output_tokens_details": {"thinking_tokens": 155}, + }, "content": [ { "type": "thinking", @@ -505,7 +515,12 @@ CLAUDE_OPUS_5_5_TEST_CASE: Final = TranslationTestCase( **BEDROCK_CONVERSE_MESSAGES_RESPONSE, "id": ANY, "model": "bedrock/converse/us.anthropic.claude-opus-5-5", - "usage": {"input_tokens": 4, "output_tokens": 190, "cache_creation_input_tokens": 853}, + "usage": { + "input_tokens": 4, + "output_tokens": 190, + "cache_creation_input_tokens": 853, + "output_tokens_details": {"thinking_tokens": 35}, + }, "content": [ { "type": "thinking", diff --git a/tests/integration/translation/messages/bases/bedrock_mantle.py b/tests/integration/translation/messages/bases/bedrock_mantle.py index 7524bf8f7ea..d9c212d1220 100644 --- a/tests/integration/translation/messages/bases/bedrock_mantle.py +++ b/tests/integration/translation/messages/bases/bedrock_mantle.py @@ -534,7 +534,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, "id": "resp_n4higs7hpzdjowbay2ujeesbgcxavsl4pbkzhbh7pjpimjjaciaa", "model": "bedrock_mantle/openai.gpt-6.1-sol", - "usage": {"input_tokens": 309, "output_tokens": 141}, + "usage": {"input_tokens": 309, "output_tokens": 141, "output_tokens_details": {"thinking_tokens": 76}}, "content": [ { "type": "text", @@ -620,7 +620,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, "id": "resp_di3jhnf3mehuuht6smddkuygfeo7rzxdxmwvrc52aigdygbts5ca", "model": "bedrock_mantle/openai.gpt-6-luna", - "usage": {"input_tokens": 309, "output_tokens": 119}, + "usage": {"input_tokens": 309, "output_tokens": 119, "output_tokens_details": {"thinking_tokens": 54}}, "content": [ { "type": "text", @@ -706,7 +706,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, "id": "resp_eznficwibpblmnocp4vuvti6d5bxboyxws4vqf5bk2lboyvvbx7q", "model": "bedrock_mantle/openai.gpt-6-astra", - "usage": {"input_tokens": 309, "output_tokens": 138}, + "usage": {"input_tokens": 309, "output_tokens": 138, "output_tokens_details": {"thinking_tokens": 73}}, "content": [ { "type": "text", @@ -793,7 +793,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( **BEDROCK_MANTLE_OPENAI_RESPONSES_MESSAGES_RESPONSE, "id": "resp_fqwklh5v6lnfftfaq6ddfbclshh3vkgmth75hufjjxjrhn726b2a", "model": "bedrock_mantle/openai.gpt-5.6-terra", - "usage": {"input_tokens": 309, "output_tokens": 109}, + "usage": {"input_tokens": 309, "output_tokens": 109, "output_tokens_details": {"thinking_tokens": 23}}, "content": [ { "type": "text", diff --git a/tests/integration/translation/messages/bases/gemini.py b/tests/integration/translation/messages/bases/gemini.py index 00d4a7fe5d2..55563559d20 100644 --- a/tests/integration/translation/messages/bases/gemini.py +++ b/tests/integration/translation/messages/bases/gemini.py @@ -152,6 +152,7 @@ GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( "usage": { "input_tokens": 354, "output_tokens": 419, + "output_tokens_details": {"thinking_tokens": 355}, }, }, ) @@ -286,6 +287,7 @@ GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( "usage": { "input_tokens": 354, "output_tokens": 302, + "output_tokens_details": {"thinking_tokens": 233}, }, }, ) @@ -416,6 +418,7 @@ GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( "usage": { "input_tokens": 412, "output_tokens": 192, + "output_tokens_details": {"thinking_tokens": 124}, }, }, ) @@ -467,7 +470,7 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( "role": "assistant", "model": "gemini/gemini-3.5-flash", "stop_sequence": None, - "usage": {"input_tokens": 10, "output_tokens": 127}, + "usage": {"input_tokens": 10, "output_tokens": 127, "output_tokens_details": {"thinking_tokens": 125}}, "content": [{"type": "text", "text": "Hello."}], "stop_reason": "end_turn", "stop_details": None, diff --git a/tests/integration/translation/messages/bases/openai.py b/tests/integration/translation/messages/bases/openai.py index f53baeafec4..503706935f0 100644 --- a/tests/integration/translation/messages/bases/openai.py +++ b/tests/integration/translation/messages/bases/openai.py @@ -394,7 +394,7 @@ GPT_6_LUNA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0ZmY4OWU3OTgyMGFkZDVjMmM1ZmZlM2MyODk4ZDNkOGEzZDczY2IxY2ExN2JmZGMxYTRmNTRlZmFlNzhkMTIzO3Jlc3BvbnNlX2lkOnJlc3BfMDNlYzQ5MWM1Y2FhZTAyOTAwNmFkMDIzOWMwODM0ODdkMDk5YTIxZDYwYmU2YjVmZGU=", "model": "openai/gpt-6-luna", - "usage": {"input_tokens": 309, "output_tokens": 214}, + "usage": {"input_tokens": 309, "output_tokens": 214, "output_tokens_details": {"thinking_tokens": 150}}, "content": [ { "type": "redacted_thinking", @@ -517,7 +517,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphMTYxN2NmYzNlODRjNzYxNDdjNTgxMmFhMjIwNGU0MDczOTM0M2YyMTg4YzQxNzIxMzRlOTc1ZTQ0YzQ3MjgyO3Jlc3BvbnNlX2lkOnJlc3BfMGFmNjdjYjc4YmVhZDJjNjAwNmFkMDIzYTcyNzM0ODdkMGIxNzY4Yjc3ZjY5NTZiNzM=", "model": "openai/gpt-6.1-sol", - "usage": {"input_tokens": 309, "output_tokens": 140}, + "usage": {"input_tokens": 309, "output_tokens": 140, "output_tokens_details": {"thinking_tokens": 75}}, "content": [ { "type": "redacted_thinking", @@ -640,7 +640,7 @@ GPT_6_ASTRA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDphODRkZDk3MTcwNzk0ZDlhYzQ5ODhlZWMzOTJmN2Q5YjQwN2MyNDY1YmJhZTA4MGE1ZGYzMThlOTViOTA2NzlkO3Jlc3BvbnNlX2lkOnJlc3BfMDk2NGE0MWY1ODE1M2FjNTAwNmFkMDIzOGY5YmMwODdkMGE5ZTc5ZmRhMzc1YmVlM2M=", "model": "openai/gpt-6-astra", - "usage": {"input_tokens": 309, "output_tokens": 152}, + "usage": {"input_tokens": 309, "output_tokens": 152, "output_tokens_details": {"thinking_tokens": 87}}, "content": [ { "type": "redacted_thinking", @@ -764,7 +764,7 @@ GPT_5_6_TERRA_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo0Mjg5MWEzMzRjNTlhMjhlMjQ2MjQ0YTQ4YTJjOGVlMTM5NGYyZGNhZWMxYmVjZDhmZDk2N2NmNjYxNWNmZDM5O3Jlc3BvbnNlX2lkOnJlc3BfMDg1MjZjY2FiOWFhNjUzZTAwNmFkMDIzODg0YTFjODdkMDlhMjNmNmQ1YzYxZDY1ZDM=", "model": "openai/gpt-5.6-terra", - "usage": {"input_tokens": 309, "output_tokens": 107}, + "usage": {"input_tokens": 309, "output_tokens": 107, "output_tokens_details": {"thinking_tokens": 42}}, "content": [ { "type": "redacted_thinking", diff --git a/tests/integration/translation/messages/bases/openai_responses.py b/tests/integration/translation/messages/bases/openai_responses.py index f5a26684ea6..c63d21fa909 100644 --- a/tests/integration/translation/messages/bases/openai_responses.py +++ b/tests/integration/translation/messages/bases/openai_responses.py @@ -201,7 +201,7 @@ GPT_6_1_SOL_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo3ZGE3NDY3Njc3N2IyZjVkYmQ3MDI1MTI1ZjEzMDdlYWQ0MjM3YTgzN2QxZTJmZDhkY2FkNjhlODM4ODU1Zjg2O3Jlc3BvbnNlX2lkOnJlc3BfMGFmNjdjYjc4YmVhZDJjNjAwNmFkMDIzYTcyNzM0ODdkMGIxNzY4Yjc3ZjY5NTZiNzM=", "model": "openai/responses/gpt-6.1-sol", - "usage": {"input_tokens": 309, "output_tokens": 140}, + "usage": {"input_tokens": 309, "output_tokens": 140, "output_tokens_details": {"thinking_tokens": 75}}, "content": [ { "type": "redacted_thinking", diff --git a/tests/integration/translation/messages/bases/vertex_ai.py b/tests/integration/translation/messages/bases/vertex_ai.py index 120b764a63c..10067668587 100644 --- a/tests/integration/translation/messages/bases/vertex_ai.py +++ b/tests/integration/translation/messages/bases/vertex_ai.py @@ -295,7 +295,7 @@ GEMINI_3_5_FLASH_TEST_CASE: Final = TranslationTestCase( "role": "assistant", "model": "vertex_ai/gemini-3.5-flash", "stop_sequence": None, - "usage": {"input_tokens": 9, "output_tokens": 86}, + "usage": {"input_tokens": 9, "output_tokens": 86, "output_tokens_details": {"thinking_tokens": 84}}, "content": [{"type": "text", "text": "Hello."}], "stop_reason": "end_turn", "stop_details": None, @@ -845,7 +845,7 @@ GEMINI_3_8_FLASH_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "ySjQarHoLb2apucPwee_YQ", "model": "vertex_ai/gemini-3.8-flash", - "usage": {"input_tokens": 242, "output_tokens": 314}, + "usage": {"input_tokens": 242, "output_tokens": 314, "output_tokens_details": {"thinking_tokens": 243}}, "content": [ { "type": "thinking", @@ -973,7 +973,7 @@ GEMINI_3_1_PRO_PREVIEW_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "xCjQasWfKcyL9LsPx7HAqQ8", "model": "vertex_ai/gemini-3.1-pro-preview", - "usage": {"input_tokens": 242, "output_tokens": 257}, + "usage": {"input_tokens": 242, "output_tokens": 257, "output_tokens_details": {"thinking_tokens": 198}}, "content": [ { "type": "thinking", @@ -1097,7 +1097,7 @@ GEMINI_4_ARGON_TEST_CASE: Final = TranslationTestCase( **MESSAGES_BRIDGE_RESPONSE, "id": "0CjQauKQPNyFpucPldSPiAY", "model": "vertex_ai/gemini-4-argon", - "usage": {"input_tokens": 412, "output_tokens": 192}, + "usage": {"input_tokens": 412, "output_tokens": 192, "output_tokens_details": {"thinking_tokens": 123}}, "content": [ { "type": "thinking", diff --git a/tests/integration/translation/messages/params/__init__.py b/tests/integration/translation/messages/params/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/messages/params/test_messages_usage_thinking_tokens.py b/tests/integration/translation/messages/params/test_messages_usage_thinking_tokens.py new file mode 100644 index 00000000000..5b1e26b5964 --- /dev/null +++ b/tests/integration/translation/messages/params/test_messages_usage_thinking_tokens.py @@ -0,0 +1,67 @@ +from dataclasses import replace +from typing import Final + +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.messages.bases.azure import GPT_5_4_TEST_CASE as AZURE_GPT_5_4_TEST_CASE +from integration.translation.messages.bases.gemini import GEMINI_3_5_FLASH_TEST_CASE +from integration.translation.messages.bases.openai import GPT_5_4_TEST_CASE as OPENAI_GPT_5_4_TEST_CASE +from integration.translation.params import with_field +from integration.translation.runner import assert_translation + +SCENARIO: Final = "usage_thinking_tokens" +THINKING_TOKENS: Final = 4 +EXPECTED_OUTPUT_TOKENS_DETAILS: Final = {"thinking_tokens": THINKING_TOKENS} + +OPENAI_GPT_5_4_THINKING_TOKENS_TEST_CASE: Final = replace( + OPENAI_GPT_5_4_TEST_CASE, + scenario=SCENARIO, + mock_provider_response=with_field( + OPENAI_GPT_5_4_TEST_CASE.mock_provider_response, + ("usage", "output_tokens_details", "reasoning_tokens"), + THINKING_TOKENS, + ), + expected_litellm_response=with_field( + OPENAI_GPT_5_4_TEST_CASE.expected_litellm_response, + ("usage", "output_tokens_details"), + EXPECTED_OUTPUT_TOKENS_DETAILS, + ), +) + +AZURE_GPT_5_4_THINKING_TOKENS_TEST_CASE: Final = replace( + AZURE_GPT_5_4_TEST_CASE, + scenario=SCENARIO, + mock_provider_response=with_field( + AZURE_GPT_5_4_TEST_CASE.mock_provider_response, + ("usage", "completion_tokens_details", "reasoning_tokens"), + THINKING_TOKENS, + ), + expected_litellm_response=with_field( + AZURE_GPT_5_4_TEST_CASE.expected_litellm_response, + ("usage", "output_tokens_details"), + EXPECTED_OUTPUT_TOKENS_DETAILS, + ), +) + +GEMINI_3_5_FLASH_THINKING_TOKENS_TEST_CASE: Final = replace( + GEMINI_3_5_FLASH_TEST_CASE, + scenario=SCENARIO, + expected_litellm_response=with_field( + GEMINI_3_5_FLASH_TEST_CASE.expected_litellm_response, + ("usage", "output_tokens_details"), + {"thinking_tokens": 125}, + ), +) + +CASES: Final = ( + OPENAI_GPT_5_4_THINKING_TOKENS_TEST_CASE, + AZURE_GPT_5_4_THINKING_TOKENS_TEST_CASE, + GEMINI_3_5_FLASH_THINKING_TOKENS_TEST_CASE, +) + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.id) +def test_messages_usage_thinking_tokens(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py b/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py index c350d6e2a10..0c60d5d5e84 100644 --- a/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py +++ b/tests/integration/translation/messages/reasoning/test_messages_reasoning_gemini.py @@ -285,7 +285,7 @@ GEMINI_3_8_FLASH_THOUGHTS_OFF_TEST_CASE: Final = replace( "provider_specific_fields": {"signature": GEMINI_3_8_FLASH_THOUGHTS_OFF_THOUGHT_SIGNATURE}, }, ], - "usage": {"input_tokens": 354, "output_tokens": 385}, + "usage": {"input_tokens": 354, "output_tokens": 385, "output_tokens_details": {"thinking_tokens": 321}}, }, ) @@ -370,4 +370,5 @@ def test_a_streamed_turn_keeps_thinking_and_billing_reasoning_without_thought_te assert [object_value(event["usage"]) for event in events if event.get("type") == "message_delta"][-1] == { "input_tokens": 354, "output_tokens": 360, + "output_tokens_details": {"thinking_tokens": 295}, } diff --git a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py index b457d9920bd..2d17e30a369 100644 --- a/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py +++ b/tests/integration/translation/messages/tool_search/test_messages_tool_search_bedrock_chat_completions.py @@ -134,7 +134,7 @@ GROK_4_7_TOOL_SEARCH_TEST_CASE: Final = replace( expected_litellm_response={ **GROK_4_7_TEST_CASE.expected_litellm_response, "id": "chatcmpl-xevjeh2pxs6gjvcn2yz76piwdc4pi6fb2sqwb7psnwitm5higvvq", - "usage": {"input_tokens": 162, "output_tokens": 76}, + "usage": {"input_tokens": 162, "output_tokens": 76, "output_tokens_details": {"thinking_tokens": 57}}, "content": [{"type": "text", "text": "The web search tool is now loaded and available for use."}], }, ) diff --git a/tests/integration/translation/params.py b/tests/integration/translation/params.py new file mode 100644 index 00000000000..30681d3aa71 --- /dev/null +++ b/tests/integration/translation/params.py @@ -0,0 +1,13 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import JsonValue + + +def with_field(mapping: Mapping[str, JsonValue], path: Sequence[str], value: JsonValue) -> Mapping[str, JsonValue]: + """Return a copy of `mapping` with the nested key `path` set to `value`, creating intermediate objects.""" + head: Final = path[0] + if len(path) == 1: + return {**mapping, head: value} + inner: Final = mapping.get(head) + return {**mapping, head: with_field(inner if isinstance(inner, dict) else {}, path[1:], value)} diff --git a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py index 420de9cb4f5..e9317043c21 100644 --- a/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py +++ b/tests/unit/llms/anthropic/messages/adapters/test_anthropic_messages_adapters_transformation.py @@ -40,6 +40,7 @@ from litellm.types.llms.openai import ChatCompletionAssistantToolCall from litellm.types.utils import ( ChatCompletionDeltaToolCall, Choices, + CompletionTokensDetailsWrapper, Delta, Function, Message, @@ -3452,6 +3453,28 @@ def test_translate_openai_usage_to_anthropic_cache_tokens_from_dict_details_with assert anthropic_usage["cache_creation_input_tokens"] == 20 +def test_translate_openai_usage_to_anthropic_maps_reasoning_tokens_to_thinking_tokens(): + usage: Final = Usage( + prompt_tokens=19, + completion_tokens=30, + total_tokens=49, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=25), + ) + + anthropic_usage: Final = LiteLLMAnthropicMessagesAdapter.translate_openai_usage_to_anthropic_usage_delta(usage) + + assert anthropic_usage["output_tokens"] == 30 + assert anthropic_usage["output_tokens_details"] == {"thinking_tokens": 25} + + +def test_translate_openai_usage_to_anthropic_omits_output_tokens_details_without_reasoning_tokens(): + usage: Final = Usage(prompt_tokens=19, completion_tokens=5, total_tokens=24) + + anthropic_usage: Final = LiteLLMAnthropicMessagesAdapter.translate_openai_usage_to_anthropic_usage_delta(usage) + + assert "output_tokens_details" not in anthropic_usage + + def test_translate_openai_usage_to_anthropic_ignores_fractional_cache_tokens(): usage = Usage( prompt_tokens=120,