From e1b8a263320e0318b2d603df47a007975699b485 Mon Sep 17 00:00:00 2001 From: qxyuan853 <142065397+qxyuan853@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:51:28 +0000 Subject: [PATCH] fix(anthropic): price advisor tool iterations at the advisor model's rate Anthropic runs each advisor tool call as a separate sub-inference on the advisor model and reports it as an advisor_message entry in usage.iterations, billed at the advisor model's rates. calculate_usage added every iteration to the request model's token counts, so advisor tokens were priced at the request model's rates and the recorded cost was too low. It now keeps advisor_message iterations out of the request model's counts and cost_per_token prices each of them at the rate of the model the iteration names. The stream chunk builder keeps usage.iterations so streamed responses are priced the same way, and the key model allowlist is also checked for the model named in each advisor tool --- .../streaming_chunk_builder_utils.py | 18 ++++++ litellm/llms/anthropic/chat/transformation.py | 30 ++++++--- litellm/llms/anthropic/cost_calculation.py | 39 +++++++++++- litellm/proxy/auth/user_api_key_auth.py | 23 +++++++ .../test_streaming_chunk_builder_utils.py | 20 ++++++ .../test_anthropic_chat_transformation.py | 63 +++++++++++++++++++ .../test_router_override_fallback_auth.py | 36 ++++++++++- 7 files changed, 219 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index d505192184e..9566a7527dc 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1018,6 +1018,21 @@ class ChunkProcessor: ] return values[-1] if values else None + def _last_usage_iterations( + self, + chunks: Sequence["_UsageBearingChunk | ModelResponse"], + ) -> list | None: + """Last ``iterations`` list reported on a usage chunk (Anthropic sends it cumulatively on the final + ``message_delta``).""" + values: Final = [ + value + for chunk in chunks + if (usage_chunk := self._extract_usage_chunk(chunk)) is not None + and isinstance(value := getattr(usage_chunk, "iterations", None), list) + and len(value) > 0 + ] + return values[-1] if values else None + @staticmethod def _finish_reason_of_choice(choice: object) -> str | None: match choice: @@ -1193,6 +1208,9 @@ class ChunkProcessor: ) if value is not None } + usage_iterations: Final = self._last_usage_iterations(chunks) + if usage_iterations is not None: + provider_pricing_fields["iterations"] = usage_iterations returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 38af6e7998b..5b9116d7c69 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -250,6 +250,8 @@ def _validated_defer_loading(value: object) -> bool: class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" + type: ReadOnly[str] + model: ReadOnly[str] input_tokens: ReadOnly[int | None] output_tokens: ReadOnly[int | None] cache_creation_input_tokens: ReadOnly[int | None] @@ -2700,9 +2702,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ephemeral_1h_input_tokens=detailed_1h, ) + @staticmethod + def executor_billed_iterations(iterations: Sequence[Any]) -> list[Any]: + """Iterations billed at the request model's rates. ``advisor_message`` iterations ran on the advisor + model and are priced separately by ``advisor_iterations_cost``.""" + return [it for it in iterations if not (isinstance(it, Mapping) and it.get("type") == "advisor_message")] + @staticmethod def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None: - iterations: Final = usage.get("iterations") + all_iterations: Final = usage.get("iterations") + iterations: Final = AnthropicConfig.executor_billed_iterations(all_iterations) if all_iterations else None if iterations: aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations) if aggregated is not None: @@ -2744,14 +2753,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations") - if iterations: - prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations) - completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations) - cache_creation_input_tokens = sum(it.get("cache_creation_input_tokens", 0) or 0 for it in iterations) - cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in iterations) + executor_iterations: Final[Sequence[_AnthropicUsageIteration] | None] = ( + self.executor_billed_iterations(iterations) if iterations else None + ) + if executor_iterations: + prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in executor_iterations) + completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in executor_iterations) + cache_creation_input_tokens = sum( + it.get("cache_creation_input_tokens", 0) or 0 for it in executor_iterations + ) + cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in executor_iterations) prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens - if not iterations: + if not executor_iterations: if "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None: cache_creation_input_tokens = _usage["cache_creation_input_tokens"] prompt_tokens += cache_creation_input_tokens @@ -2791,7 +2805,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) completion_token_details: Final = self._build_completion_token_details( usage_object=_usage, - iterations=iterations, + iterations=executor_iterations, completion_tokens=completion_tokens, reasoning_content=reasoning_content, completion_response=completion_response, diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 94c8149eff3..14b0f2e46fc 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -3,6 +3,7 @@ Helper util for handling anthropic-specific cost calculation - e.g.: prompt caching """ +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional from pydantic import ValidationError @@ -64,7 +65,43 @@ def cost_per_token( except Exception: pass - return prompt_cost, completion_cost + advisor_prompt_cost, advisor_completion_cost = advisor_iterations_cost(usage) + return prompt_cost + advisor_prompt_cost, completion_cost + advisor_completion_cost + + +def advisor_iterations_cost(usage: "Usage") -> tuple[float, float]: + """Prices each ``advisor_message`` iteration at the rate of the advisor model it names. Raises for an + iteration without a model or whose model has no cost map entry.""" + from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + def iteration_cost(iteration: Mapping[str, object]) -> tuple[float, float]: + advisor_model: Final = iteration.get("model") + if not isinstance(advisor_model, str) or not advisor_model: + raise ValueError("Anthropic advisor_message iteration has no model, so it cannot be priced") + litellm.get_model_info(model=advisor_model, custom_llm_provider="anthropic") + cache_read: Final = int(iteration.get("cache_read_input_tokens") or 0) + cache_write: Final = int(iteration.get("cache_creation_input_tokens") or 0) + text_input: Final = int(iteration.get("input_tokens") or 0) + return generic_cost_per_token( + model=advisor_model, + usage=Usage( + prompt_tokens=text_input + cache_read + cache_write, + completion_tokens=int(iteration.get("output_tokens") or 0), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cache_read, cache_creation_tokens=cache_write, text_tokens=text_input + ), + cache_creation_input_tokens=cache_write, + cache_read_input_tokens=cache_read, + ), + custom_llm_provider="anthropic", + ) + + costs: Final = tuple( + iteration_cost(iteration) + for iteration in getattr(usage, "iterations", None) or () + if isinstance(iteration, Mapping) and iteration.get("type") == "advisor_message" + ) + return sum(prompt for prompt, _ in costs), sum(completion for _, completion in costs) class _AnthropicServerToolUseProbe(LiteLLMBaseModel): diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index a1abed202a2..29dacc73280 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -4514,6 +4514,29 @@ async def enforce_key_and_fallback_model_access( user_model=None, ) + for _advisor_model in dict.fromkeys(advisor_tool_models(request_data)): + await can_key_call_model( + model=_advisor_model, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + + +def advisor_tool_models(request_data: Mapping[str, object]) -> list[str]: + """Models named by the request's ``advisor_20260301`` tools. A non-string model is returned as its repr so + the allowlist check denies it instead of skipping it.""" + from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE + + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return [] + return [ + model if isinstance(model := tool.get("model"), str) else repr(model) + for tool in tools + if isinstance(tool, Mapping) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE + ] + _enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index f8cd0416b1c..c69dad706a3 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -2180,3 +2180,23 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items + + +def test_chunk_builder_keeps_the_final_usage_iterations_for_cost_calculation(): + iterations: Final = [ + {"type": "message", "input_tokens": 412, "output_tokens": 89}, + {"type": "advisor_message", "model": "claude-opus-4-8", "input_tokens": 823, "output_tokens": 1612}, + ] + chunks: Final = [ + ModelResponseStream(choices=[], usage=Usage(prompt_tokens=412, completion_tokens=1, total_tokens=413)), + ModelResponseStream( + choices=[], + usage=Usage(prompt_tokens=412, completion_tokens=89, total_tokens=501, iterations=iterations), + ), + ] + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="claude-haiku-4-5", completion_output="" + ) + + assert usage.iterations == iterations diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index 6323132db36..ed27afceb18 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -25,6 +25,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token +from litellm.exceptions import ModelNotMappedError from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -9000,6 +9002,67 @@ def test_calculate_usage_sums_cache_tokens_across_compaction_iterations(): assert usage.prompt_tokens_details.cached_tokens == 17020 +ADVISOR_EXECUTOR_MODEL: Final = "claude-haiku-4-5" +ADVISOR_MODEL: Final = "claude-opus-4-8" + + +def _advisor_usage_object(advisor_model: object = ADVISOR_MODEL) -> dict[str, object]: + return { + "input_tokens": 1760, + "cache_read_input_tokens": 412, + "cache_creation_input_tokens": 0, + "output_tokens": 531, + "iterations": [ + {"type": "message", "input_tokens": 412, "output_tokens": 89}, + {"type": "advisor_message", "model": advisor_model, "input_tokens": 823, "output_tokens": 1612}, + { + "type": "message", + "input_tokens": 1348, + "cache_read_input_tokens": 412, + "output_tokens": 442, + }, + ], + } + + +def test_calculate_usage_keeps_advisor_iterations_out_of_the_request_model_tokens(): + usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None) + assert usage.prompt_tokens == 1760 + 412 + assert usage.completion_tokens == 531 + assert usage.prompt_tokens_details.cached_tokens == 412 + assert [iteration["type"] for iteration in usage.iterations] == ["message", "advisor_message", "message"] + + +def test_cost_per_token_prices_advisor_iterations_at_the_advisor_models_rate(): + usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None) + executor: Final = litellm.get_model_info(model=ADVISOR_EXECUTOR_MODEL, custom_llm_provider="anthropic") + advisor: Final = litellm.get_model_info(model=ADVISOR_MODEL, custom_llm_provider="anthropic") + + prompt_cost, completion_cost = anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage) + + executor_cost: Final = ( + 1760 * executor["input_cost_per_token"] + + 412 * executor["cache_read_input_token_cost"] + + 531 * executor["output_cost_per_token"] + ) + advisor_cost: Final = 823 * advisor["input_cost_per_token"] + 1612 * advisor["output_cost_per_token"] + assert prompt_cost + completion_cost == pytest.approx(executor_cost + advisor_cost, rel=1e-9) + + +@pytest.mark.parametrize( + "advisor_model, expected_error", + [("claude-no-such-advisor", ModelNotMappedError), (None, ValueError)], +) +def test_cost_per_token_rejects_an_advisor_iteration_it_cannot_price( + advisor_model: object, expected_error: type[Exception] +): + usage: Final = AnthropicConfig().calculate_usage( + usage_object=_advisor_usage_object(advisor_model), reasoning_content=None + ) + with pytest.raises(expected_error): + anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage) + + PROMPT_CACHING_MODEL: Final = "claude-sonnet-5-5" EPHEMERAL: Final = {"type": "ephemeral"} WEATHER_PARAMETERS: Final = { diff --git a/tests/unit/proxy/auth/test_router_override_fallback_auth.py b/tests/unit/proxy/auth/test_router_override_fallback_auth.py index d35a5851659..614d63c4dd4 100644 --- a/tests/unit/proxy/auth/test_router_override_fallback_auth.py +++ b/tests/unit/proxy/auth/test_router_override_fallback_auth.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch import pytest -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import fallback_target_model_name, iter_request_fallback_targets from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access @@ -337,3 +337,37 @@ async def test_router_override_without_fallbacks_does_not_break_auth(): ) assert seen == ["gpt-3.5-turbo"] + + +def _advisor_request(advisor_model: str) -> dict: + return { + "model": "cheap-executor", + "tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor_model}], + } + + +@pytest.mark.asyncio +async def test_advisor_tool_model_outside_the_key_allowlist_is_denied(): + with pytest.raises(ProxyException) as denial: + await enforce_key_and_fallback_model_access( + valid_token=_key_with_models(["cheap-executor"]), + request_data=_advisor_request("premium-advisor"), + route="/v1/messages", + request=None, + llm_model_list=None, + llm_router=None, + ) + + assert denial.value.type == ProxyErrorTypes.key_model_access_denied + + +@pytest.mark.asyncio +async def test_advisor_tool_model_the_key_may_call_is_admitted(): + await enforce_key_and_fallback_model_access( + valid_token=_key_with_models(["cheap-executor", "premium-advisor"]), + request_data=_advisor_request("premium-advisor"), + route="/v1/messages", + request=None, + llm_model_list=None, + llm_router=None, + )