diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json index 1b0650c70d8..8e3bbe78327 100644 --- a/basedpyright-code-budget.json +++ b/basedpyright-code-budget.json @@ -12,7 +12,7 @@ "limit": 480 }, "reportCallIssue": { - "limit": 112 + "limit": 110 }, "reportConstantRedefinition": { "limit": 40 diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 5f7ac73c919..4a4873216b9 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -2253,17 +2253,22 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): @staticmethod def is_anthropic_usage_object(usage_object: dict) -> bool: """Anthropic reports prompt cache tokens as top-level ``cache_read_input_tokens`` / - ``cache_creation_input_tokens``; no other API surface uses those keys, and the - Responses API mapping would silently drop them. + ``cache_creation_input_tokens`` and thinking tokens as + ``output_tokens_details.thinking_tokens``; no other API surface uses those keys, and + the Responses API mapping would silently drop both (it only reads + ``output_tokens_details.reasoning_tokens``). - Requiring a cache key is deliberate: Responses API usage also carries top-level - ``input_tokens``, so the cache keys are the only shape discriminator between the - two. A cache-free Anthropic payload falls through to the Responses API mapping, - which is safe because both mappings agree whenever no cache tokens are present. + Requiring one of those keys is deliberate: Responses API usage also carries top-level + ``input_tokens``, so they are the only shape discriminators between the two. An + Anthropic payload carrying none of them falls through to the Responses API mapping, + which is safe because both mappings agree whenever no cache or thinking tokens are + present. """ if "prompt_tokens" in usage_object or "input_tokens" not in usage_object: return False - return any(key in usage_object for key in ("cache_read_input_tokens", "cache_creation_input_tokens")) + if any(key in usage_object for key in ("cache_read_input_tokens", "cache_creation_input_tokens")): + return True + return AnthropicConfig.thinking_tokens_from_usage(usage_object) is not None @staticmethod def _aggregate_cache_creation_token_details( diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 573a461e89e..f118f532109 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -138,6 +138,7 @@ from litellm.types.llms.anthropic import ( ContextManagementResponse, MessageBlockDelta, MessageDelta, + OutputTokensDetailsUsage, ServerToolUsage, StreamingContentBlockDeltaType, UsageDelta, @@ -1429,11 +1430,20 @@ class LiteLLMAnthropicMessagesAdapter: return from_server_tool_use return cls._first_positive_prompt_tokens_detail_value(usage, ("web_search_requests",)) + @classmethod + def _get_reasoning_token_count(cls, usage: Usage) -> int: + from litellm.litellm_core_utils.llm_cost_calc.utils import ( + parse_completion_tokens_details, + ) + + return cls._positive_int(parse_completion_tokens_details(usage)["reasoning_tokens"]) + @classmethod def _translate_openai_usage_to_anthropic_usage_delta(cls, usage: Usage) -> UsageDelta: cache_read_input_tokens: Final = cls._get_cache_read_input_tokens(usage) cache_creation_input_tokens: Final = cls._get_cache_creation_input_tokens(usage) web_search_requests: Final = cls._get_web_search_request_count(usage) + thinking_tokens: Final = cls._get_reasoning_token_count(usage) input_tokens: Final = max( (usage.prompt_tokens or 0) - cache_read_input_tokens - cache_creation_input_tokens, 0, @@ -1447,12 +1457,30 @@ class LiteLLMAnthropicMessagesAdapter: usage_delta["cache_creation_input_tokens"] = cache_creation_input_tokens if cache_read_input_tokens > 0: usage_delta["cache_read_input_tokens"] = cache_read_input_tokens - if web_search_requests > 0: - return UsageDelta( - **usage_delta, - server_tool_use=ServerToolUsage(web_search_requests=web_search_requests), - ) - return usage_delta + return cls._with_thinking_tokens( + cls._with_server_tool_use(usage_delta, web_search_requests), + thinking_tokens, + ) + + @staticmethod + def _with_server_tool_use(usage_delta: UsageDelta, web_search_requests: int) -> UsageDelta: + if web_search_requests == 0: + return usage_delta + with_server_tool_use: Final[UsageDelta] = { + **usage_delta, + "server_tool_use": ServerToolUsage(web_search_requests=web_search_requests), + } + return with_server_tool_use + + @staticmethod + def _with_thinking_tokens(usage_delta: UsageDelta, thinking_tokens: int) -> UsageDelta: + if thinking_tokens == 0: + return usage_delta + with_thinking_tokens: Final[UsageDelta] = { + **usage_delta, + "output_tokens_details": OutputTokensDetailsUsage(thinking_tokens=thinking_tokens), + } + return with_thinking_tokens @classmethod def _translate_openai_usage_to_anthropic_usage(cls, usage: Usage) -> AnthropicUsage: diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index b3462203c4b..631c471c333 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -528,12 +528,17 @@ class ServerToolUsage(TypedDict, total=False): web_search_requests: ReadOnly[int] +class OutputTokensDetailsUsage(TypedDict, total=False): + thinking_tokens: ReadOnly[int] + + class UsageDelta(TypedDict, total=False): input_tokens: int output_tokens: int cache_creation_input_tokens: int cache_read_input_tokens: int server_tool_use: ReadOnly[ServerToolUsage] + output_tokens_details: ReadOnly[OutputTokensDetailsUsage] class AppliedEdit(TypedDict, total=False): diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 4fe1dafc73b..6ee47eaf577 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -6,6 +6,7 @@ from litellm.types.llms.anthropic import ( AnthropicResponseContentBlockText, AnthropicResponseContentBlockToolUse, ContextManagementResponse, + OutputTokensDetailsUsage, ServerToolUsage, ) @@ -77,6 +78,11 @@ class AnthropicUsage(TypedDict, total=False): """ server_tool_use: NotRequired[ReadOnly[ServerToolUsage]] + """ + Output token breakdown (thinking tokens) + """ + output_tokens_details: NotRequired[ReadOnly[OutputTokensDetailsUsage]] + class AnthropicStopDetails(TypedDict, total=False): """ diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 8ea8db5fb65..7aaef753c31 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -6409,3 +6409,18 @@ def test_response_format_tool_path_skips_forced_tool_choice_when_unsupported(loc assert "tools" in result assert "tool_choice" not in result + + +def test_is_anthropic_usage_object_accepts_cache_free_usage_reporting_thinking_tokens(): + """Only Anthropic-shaped usage nests ``thinking_tokens`` under ``output_tokens_details``, + so a cache-free payload carrying it must take the Anthropic mapping, which is the only + converter that turns the count into billable reasoning tokens.""" + assert AnthropicConfig.is_anthropic_usage_object( + {"input_tokens": 105, "output_tokens": 77, "output_tokens_details": {"thinking_tokens": 75}} + ) + + +def test_is_anthropic_usage_object_rejects_cache_free_responses_api_reasoning_tokens(): + assert not AnthropicConfig.is_anthropic_usage_object( + {"input_tokens": 105, "output_tokens": 77, "output_tokens_details": {"reasoning_tokens": 75}} + ) diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py index 2d74c00071b..15652abaf1a 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py @@ -4630,3 +4630,118 @@ def test_a_bedrock_target_still_takes_output_config_not_the_declared_gate(): assert openai_request["output_config"] == {"effort": "max"} assert "reasoning_effort" not in openai_request assert openai_request["thinking"] == {"type": "adaptive", "display": "omitted"} + + +def _reasoning_model_response(model: str, reasoning_tokens: int | None) -> ModelResponse: + from litellm.types.utils import CompletionTokensDetailsWrapper + + return ModelResponse( + id="resp_reasoning", + model=model, + choices=[ + Choices( + finish_reason="stop", + message=Message( + role="assistant", + content="391", + reasoning_content="17 times 20 is 340 and 17 times 3 is 51, so the product is 391.", + ), + ) + ], + usage=Usage( + prompt_tokens=105, + completion_tokens=77, + total_tokens=182, + completion_tokens_details=( + CompletionTokensDetailsWrapper(reasoning_tokens=reasoning_tokens) + if reasoning_tokens is not None + else None + ), + ), + ) + + +def test_translate_openai_response_to_anthropic_reports_provider_reasoning_tokens_as_thinking_tokens(): + """A non-Anthropic model answering over /v1/messages reports its reasoning tokens in the + OpenAI usage; the Anthropic usage must carry that count as ``output_tokens_details.thinking_tokens`` + so cost calculation can bill them at the reasoning rate instead of the output rate.""" + anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=_reasoning_model_response(model="deepseek-v4-flash", reasoning_tokens=75) + ) + + assert anthropic_response["usage"]["output_tokens"] == 77 + assert anthropic_response["usage"]["output_tokens_details"] == {"thinking_tokens": 75} + + +@pytest.mark.parametrize("reasoning_tokens", [None, 0]) +def test_translate_openai_response_to_anthropic_omits_output_tokens_details_without_reasoning_tokens( + reasoning_tokens: int | None, +): + anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=_reasoning_model_response(model="gpt-4o-mini", reasoning_tokens=reasoning_tokens) + ) + + assert anthropic_response["usage"]["output_tokens"] == 77 + assert "output_tokens_details" not in anthropic_response["usage"] + + +def test_translated_messages_response_bills_reasoning_tokens_at_the_reasoning_rate(monkeypatch): + """The response-cost header path prices the translated Anthropic usage; with reasoning tokens + carried through, /v1/messages must cost the same as the identical /v1/chat/completions call.""" + from litellm.cost_calculator import _get_usage_object + + model = "lit6908-reasoning-model" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "input_cost_per_token": 1e-06, + "output_cost_per_token": 2e-06, + "output_cost_per_reasoning_token": 1e-05, + "litellm_provider": "deepseek", + "mode": "chat", + }, + ) + openai_response = _reasoning_model_response(model=model, reasoning_tokens=75) + anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=openai_response + ) + + normalized_usage = _get_usage_object(completion_response=anthropic_response) + assert normalized_usage is not None + assert normalized_usage.completion_tokens_details is not None + assert normalized_usage.completion_tokens_details.reasoning_tokens == 75 + + chat_cost = litellm.completion_cost(completion_response=openai_response, custom_llm_provider="deepseek") + messages_cost = litellm.completion_cost( + completion_response=anthropic_response, + model=model, + custom_llm_provider="deepseek", + call_type="anthropic_messages", + ) + assert chat_cost == pytest.approx(105 * 1e-06 + 75 * 1e-05 + 2 * 2e-06) + assert messages_cost == pytest.approx(chat_cost) + + +def test_spend_log_usage_for_translated_messages_response_uses_the_reported_thinking_tokens(): + """The spend row is built by re-parsing the translated Anthropic response; it must bill the + provider's reported reasoning count rather than a token-counter estimate of the thinking text.""" + import httpx + + from litellm.types.llms.anthropic import AnthropicResponse + + anthropic_response = LiteLLMAnthropicMessagesAdapter().translate_openai_response_to_anthropic( + response=_reasoning_model_response(model="deepseek-v4-flash", reasoning_tokens=75) + ) + assert anthropic_response["content"][0]["type"] == "thinking" + + logged = litellm.AnthropicConfig().transform_parsed_response( + completion_response=AnthropicResponse.model_validate(anthropic_response).model_dump(), + raw_response=httpx.Response(status_code=200, headers={}), + model_response=ModelResponse(), + json_mode=None, + ) + + assert logged.usage.completion_tokens == 77 + assert logged.usage.completion_tokens_details.reasoning_tokens == 75 + assert logged.usage.completion_tokens_details.text_tokens == 2 diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py index 3e85872f1e5..ed9f0506000 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_streaming_iterator_empty_choices.py @@ -85,3 +85,27 @@ def test_final_usage_chunk_without_choices_is_merged_into_message_delta(): assert message_delta["usage"]["output_tokens"] == 3 assert "Hi" in sse assert "message_stop" in sse + + +def test_final_usage_chunk_reasoning_tokens_land_on_message_delta_as_thinking_tokens(): + """A reasoning model's final usage chunk reports ``reasoning_tokens``; the Anthropic + ``message_delta`` usage must carry them as ``output_tokens_details.thinking_tokens``.""" + from litellm.types.utils import CompletionTokensDetailsWrapper + + usage = Usage( + prompt_tokens=32, + completion_tokens=662, + total_tokens=694, + completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=654), + ) + + async def _aiter() -> "AsyncIterator[ModelResponseStream]": + for chunk in [_text_chunk("391"), _finish_chunk(), _empty_choices_chunk(usage)]: + yield chunk + + sse = _collect_async(AnthropicStreamWrapper(completion_stream=_aiter(), model="mock-model")) + + message_delta = _message_delta(sse) + assert message_delta["usage"]["input_tokens"] == 32 + assert message_delta["usage"]["output_tokens"] == 662 + assert message_delta["usage"]["output_tokens_details"] == {"thinking_tokens": 654}