fix(anthropic): price advisor tool iterations at the advisor model's rate

Anthropic runs each advisor tool call as a separate sub-inference on the advisor model and reports it as an advisor_message entry in usage.iterations, billed at the advisor model's rates. calculate_usage added every iteration to the request model's token counts, so advisor tokens were priced at the request model's rates and the recorded cost was too low. It now keeps advisor_message iterations out of the request model's counts and cost_per_token prices each of them at the rate of the model the iteration names. The stream chunk builder keeps usage.iterations so streamed responses are priced the same way, and the key model allowlist is also checked for the model named in each advisor tool
This commit is contained in:
qxyuan853 2026-10-15 17:51:28 +00:00
parent caad4d6204
commit e1b8a26332
7 changed files with 219 additions and 10 deletions

View file

@ -1018,6 +1018,21 @@ class ChunkProcessor:
]
return values[-1] if values else None
def _last_usage_iterations(
self,
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
) -> list | None:
"""Last ``iterations`` list reported on a usage chunk (Anthropic sends it cumulatively on the final
``message_delta``)."""
values: Final = [
value
for chunk in chunks
if (usage_chunk := self._extract_usage_chunk(chunk)) is not None
and isinstance(value := getattr(usage_chunk, "iterations", None), list)
and len(value) > 0
]
return values[-1] if values else None
@staticmethod
def _finish_reason_of_choice(choice: object) -> str | None:
match choice:
@ -1193,6 +1208,9 @@ class ChunkProcessor:
)
if value is not None
}
usage_iterations: Final = self._last_usage_iterations(chunks)
if usage_iterations is not None:
provider_pricing_fields["iterations"] = usage_iterations
returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields)

View file

@ -250,6 +250,8 @@ def _validated_defer_loading(value: object) -> bool:
class _AnthropicUsageIteration(TypedDict, total=False):
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
type: ReadOnly[str]
model: ReadOnly[str]
input_tokens: ReadOnly[int | None]
output_tokens: ReadOnly[int | None]
cache_creation_input_tokens: ReadOnly[int | None]
@ -2700,9 +2702,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ephemeral_1h_input_tokens=detailed_1h,
)
@staticmethod
def executor_billed_iterations(iterations: Sequence[Any]) -> list[Any]:
"""Iterations billed at the request model's rates. ``advisor_message`` iterations ran on the advisor
model and are priced separately by ``advisor_iterations_cost``."""
return [it for it in iterations if not (isinstance(it, Mapping) and it.get("type") == "advisor_message")]
@staticmethod
def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None:
iterations: Final = usage.get("iterations")
all_iterations: Final = usage.get("iterations")
iterations: Final = AnthropicConfig.executor_billed_iterations(all_iterations) if all_iterations else None
if iterations:
aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations)
if aggregated is not None:
@ -2744,14 +2753,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations")
if iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations)
cache_creation_input_tokens = sum(it.get("cache_creation_input_tokens", 0) or 0 for it in iterations)
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in iterations)
executor_iterations: Final[Sequence[_AnthropicUsageIteration] | None] = (
self.executor_billed_iterations(iterations) if iterations else None
)
if executor_iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in executor_iterations)
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in executor_iterations)
cache_creation_input_tokens = sum(
it.get("cache_creation_input_tokens", 0) or 0 for it in executor_iterations
)
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in executor_iterations)
prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens
if not iterations:
if not executor_iterations:
if "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None:
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
prompt_tokens += cache_creation_input_tokens
@ -2791,7 +2805,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
completion_token_details: Final = self._build_completion_token_details(
usage_object=_usage,
iterations=iterations,
iterations=executor_iterations,
completion_tokens=completion_tokens,
reasoning_content=reasoning_content,
completion_response=completion_response,

View file

@ -3,6 +3,7 @@ Helper util for handling anthropic-specific cost calculation
- e.g.: prompt caching
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Optional
from pydantic import ValidationError
@ -64,7 +65,43 @@ def cost_per_token(
except Exception:
pass
return prompt_cost, completion_cost
advisor_prompt_cost, advisor_completion_cost = advisor_iterations_cost(usage)
return prompt_cost + advisor_prompt_cost, completion_cost + advisor_completion_cost
def advisor_iterations_cost(usage: "Usage") -> tuple[float, float]:
"""Prices each ``advisor_message`` iteration at the rate of the advisor model it names. Raises for an
iteration without a model or whose model has no cost map entry."""
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
def iteration_cost(iteration: Mapping[str, object]) -> tuple[float, float]:
advisor_model: Final = iteration.get("model")
if not isinstance(advisor_model, str) or not advisor_model:
raise ValueError("Anthropic advisor_message iteration has no model, so it cannot be priced")
litellm.get_model_info(model=advisor_model, custom_llm_provider="anthropic")
cache_read: Final = int(iteration.get("cache_read_input_tokens") or 0)
cache_write: Final = int(iteration.get("cache_creation_input_tokens") or 0)
text_input: Final = int(iteration.get("input_tokens") or 0)
return generic_cost_per_token(
model=advisor_model,
usage=Usage(
prompt_tokens=text_input + cache_read + cache_write,
completion_tokens=int(iteration.get("output_tokens") or 0),
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=cache_read, cache_creation_tokens=cache_write, text_tokens=text_input
),
cache_creation_input_tokens=cache_write,
cache_read_input_tokens=cache_read,
),
custom_llm_provider="anthropic",
)
costs: Final = tuple(
iteration_cost(iteration)
for iteration in getattr(usage, "iterations", None) or ()
if isinstance(iteration, Mapping) and iteration.get("type") == "advisor_message"
)
return sum(prompt for prompt, _ in costs), sum(completion for _, completion in costs)
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):

View file

@ -4514,6 +4514,29 @@ async def enforce_key_and_fallback_model_access(
user_model=None,
)
for _advisor_model in dict.fromkeys(advisor_tool_models(request_data)):
await can_key_call_model(
model=_advisor_model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
def advisor_tool_models(request_data: Mapping[str, object]) -> list[str]:
"""Models named by the request's ``advisor_20260301`` tools. A non-string model is returned as its repr so
the allowlist check denies it instead of skipping it."""
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
tools: Final = request_data.get("tools")
if not isinstance(tools, list):
return []
return [
model if isinstance(model := tool.get("model"), str) else repr(model)
for tool in tools
if isinstance(tool, Mapping) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE
]
_enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias

View file

@ -2180,3 +2180,23 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
assert response is not None
assert response.choices[0].message.content == "Hello world"
assert response.choices[0].message.reasoning_items == reasoning_items
def test_chunk_builder_keeps_the_final_usage_iterations_for_cost_calculation():
iterations: Final = [
{"type": "message", "input_tokens": 412, "output_tokens": 89},
{"type": "advisor_message", "model": "claude-opus-4-8", "input_tokens": 823, "output_tokens": 1612},
]
chunks: Final = [
ModelResponseStream(choices=[], usage=Usage(prompt_tokens=412, completion_tokens=1, total_tokens=413)),
ModelResponseStream(
choices=[],
usage=Usage(prompt_tokens=412, completion_tokens=89, total_tokens=501, iterations=iterations),
),
]
usage: Final = ChunkProcessor(chunks=chunks).calculate_usage(
chunks=chunks, model="claude-haiku-4-5", completion_output=""
)
assert usage.iterations == iterations

View file

@ -25,6 +25,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag
from litellm.llms.anthropic.chat import ModelResponseIterator
from litellm.llms.anthropic.common_utils import process_anthropic_headers
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
from litellm.exceptions import ModelNotMappedError
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -9000,6 +9002,67 @@ def test_calculate_usage_sums_cache_tokens_across_compaction_iterations():
assert usage.prompt_tokens_details.cached_tokens == 17020
ADVISOR_EXECUTOR_MODEL: Final = "claude-haiku-4-5"
ADVISOR_MODEL: Final = "claude-opus-4-8"
def _advisor_usage_object(advisor_model: object = ADVISOR_MODEL) -> dict[str, object]:
return {
"input_tokens": 1760,
"cache_read_input_tokens": 412,
"cache_creation_input_tokens": 0,
"output_tokens": 531,
"iterations": [
{"type": "message", "input_tokens": 412, "output_tokens": 89},
{"type": "advisor_message", "model": advisor_model, "input_tokens": 823, "output_tokens": 1612},
{
"type": "message",
"input_tokens": 1348,
"cache_read_input_tokens": 412,
"output_tokens": 442,
},
],
}
def test_calculate_usage_keeps_advisor_iterations_out_of_the_request_model_tokens():
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
assert usage.prompt_tokens == 1760 + 412
assert usage.completion_tokens == 531
assert usage.prompt_tokens_details.cached_tokens == 412
assert [iteration["type"] for iteration in usage.iterations] == ["message", "advisor_message", "message"]
def test_cost_per_token_prices_advisor_iterations_at_the_advisor_models_rate():
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
executor: Final = litellm.get_model_info(model=ADVISOR_EXECUTOR_MODEL, custom_llm_provider="anthropic")
advisor: Final = litellm.get_model_info(model=ADVISOR_MODEL, custom_llm_provider="anthropic")
prompt_cost, completion_cost = anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
executor_cost: Final = (
1760 * executor["input_cost_per_token"]
+ 412 * executor["cache_read_input_token_cost"]
+ 531 * executor["output_cost_per_token"]
)
advisor_cost: Final = 823 * advisor["input_cost_per_token"] + 1612 * advisor["output_cost_per_token"]
assert prompt_cost + completion_cost == pytest.approx(executor_cost + advisor_cost, rel=1e-9)
@pytest.mark.parametrize(
"advisor_model, expected_error",
[("claude-no-such-advisor", ModelNotMappedError), (None, ValueError)],
)
def test_cost_per_token_rejects_an_advisor_iteration_it_cannot_price(
advisor_model: object, expected_error: type[Exception]
):
usage: Final = AnthropicConfig().calculate_usage(
usage_object=_advisor_usage_object(advisor_model), reasoning_content=None
)
with pytest.raises(expected_error):
anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
PROMPT_CACHING_MODEL: Final = "claude-sonnet-5-5"
EPHEMERAL: Final = {"type": "ephemeral"}
WEATHER_PARAMETERS: Final = {

View file

@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch
import pytest
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy._types import LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import fallback_target_model_name, iter_request_fallback_targets
from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access
@ -337,3 +337,37 @@ async def test_router_override_without_fallbacks_does_not_break_auth():
)
assert seen == ["gpt-3.5-turbo"]
def _advisor_request(advisor_model: str) -> dict:
return {
"model": "cheap-executor",
"tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor_model}],
}
@pytest.mark.asyncio
async def test_advisor_tool_model_outside_the_key_allowlist_is_denied():
with pytest.raises(ProxyException) as denial:
await enforce_key_and_fallback_model_access(
valid_token=_key_with_models(["cheap-executor"]),
request_data=_advisor_request("premium-advisor"),
route="/v1/messages",
request=None,
llm_model_list=None,
llm_router=None,
)
assert denial.value.type == ProxyErrorTypes.key_model_access_denied
@pytest.mark.asyncio
async def test_advisor_tool_model_the_key_may_call_is_admitted():
await enforce_key_and_fallback_model_access(
valid_token=_key_with_models(["cheap-executor", "premium-advisor"]),
request_data=_advisor_request("premium-advisor"),
route="/v1/messages",
request=None,
llm_model_list=None,
llm_router=None,
)