mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(anthropic): price advisor tool iterations at the advisor model's rate
Anthropic runs each advisor tool call as a separate sub-inference on the advisor model and reports it as an advisor_message entry in usage.iterations, billed at the advisor model's rates. calculate_usage added every iteration to the request model's token counts, so advisor tokens were priced at the request model's rates and the recorded cost was too low. It now keeps advisor_message iterations out of the request model's counts and cost_per_token prices each of them at the rate of the model the iteration names. The stream chunk builder keeps usage.iterations so streamed responses are priced the same way, and the key model allowlist is also checked for the model named in each advisor tool
This commit is contained in:
parent
caad4d6204
commit
e1b8a26332
7 changed files with 219 additions and 10 deletions
|
|
@ -1018,6 +1018,21 @@ class ChunkProcessor:
|
|||
]
|
||||
return values[-1] if values else None
|
||||
|
||||
def _last_usage_iterations(
|
||||
self,
|
||||
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
|
||||
) -> list | None:
|
||||
"""Last ``iterations`` list reported on a usage chunk (Anthropic sends it cumulatively on the final
|
||||
``message_delta``)."""
|
||||
values: Final = [
|
||||
value
|
||||
for chunk in chunks
|
||||
if (usage_chunk := self._extract_usage_chunk(chunk)) is not None
|
||||
and isinstance(value := getattr(usage_chunk, "iterations", None), list)
|
||||
and len(value) > 0
|
||||
]
|
||||
return values[-1] if values else None
|
||||
|
||||
@staticmethod
|
||||
def _finish_reason_of_choice(choice: object) -> str | None:
|
||||
match choice:
|
||||
|
|
@ -1193,6 +1208,9 @@ class ChunkProcessor:
|
|||
)
|
||||
if value is not None
|
||||
}
|
||||
usage_iterations: Final = self._last_usage_iterations(chunks)
|
||||
if usage_iterations is not None:
|
||||
provider_pricing_fields["iterations"] = usage_iterations
|
||||
|
||||
returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields)
|
||||
|
||||
|
|
|
|||
|
|
@ -250,6 +250,8 @@ def _validated_defer_loading(value: object) -> bool:
|
|||
class _AnthropicUsageIteration(TypedDict, total=False):
|
||||
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
|
||||
|
||||
type: ReadOnly[str]
|
||||
model: ReadOnly[str]
|
||||
input_tokens: ReadOnly[int | None]
|
||||
output_tokens: ReadOnly[int | None]
|
||||
cache_creation_input_tokens: ReadOnly[int | None]
|
||||
|
|
@ -2700,9 +2702,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
ephemeral_1h_input_tokens=detailed_1h,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def executor_billed_iterations(iterations: Sequence[Any]) -> list[Any]:
|
||||
"""Iterations billed at the request model's rates. ``advisor_message`` iterations ran on the advisor
|
||||
model and are priced separately by ``advisor_iterations_cost``."""
|
||||
return [it for it in iterations if not (isinstance(it, Mapping) and it.get("type") == "advisor_message")]
|
||||
|
||||
@staticmethod
|
||||
def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None:
|
||||
iterations: Final = usage.get("iterations")
|
||||
all_iterations: Final = usage.get("iterations")
|
||||
iterations: Final = AnthropicConfig.executor_billed_iterations(all_iterations) if all_iterations else None
|
||||
if iterations:
|
||||
aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations)
|
||||
if aggregated is not None:
|
||||
|
|
@ -2744,14 +2753,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
|
||||
|
||||
iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations")
|
||||
if iterations:
|
||||
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
|
||||
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations)
|
||||
cache_creation_input_tokens = sum(it.get("cache_creation_input_tokens", 0) or 0 for it in iterations)
|
||||
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in iterations)
|
||||
executor_iterations: Final[Sequence[_AnthropicUsageIteration] | None] = (
|
||||
self.executor_billed_iterations(iterations) if iterations else None
|
||||
)
|
||||
if executor_iterations:
|
||||
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in executor_iterations)
|
||||
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in executor_iterations)
|
||||
cache_creation_input_tokens = sum(
|
||||
it.get("cache_creation_input_tokens", 0) or 0 for it in executor_iterations
|
||||
)
|
||||
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in executor_iterations)
|
||||
prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens
|
||||
|
||||
if not iterations:
|
||||
if not executor_iterations:
|
||||
if "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None:
|
||||
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
|
||||
prompt_tokens += cache_creation_input_tokens
|
||||
|
|
@ -2791,7 +2805,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
)
|
||||
completion_token_details: Final = self._build_completion_token_details(
|
||||
usage_object=_usage,
|
||||
iterations=iterations,
|
||||
iterations=executor_iterations,
|
||||
completion_tokens=completion_tokens,
|
||||
reasoning_content=reasoning_content,
|
||||
completion_response=completion_response,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Helper util for handling anthropic-specific cost calculation
|
|||
- e.g.: prompt caching
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final, Optional
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
|
@ -64,7 +65,43 @@ def cost_per_token(
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
return prompt_cost, completion_cost
|
||||
advisor_prompt_cost, advisor_completion_cost = advisor_iterations_cost(usage)
|
||||
return prompt_cost + advisor_prompt_cost, completion_cost + advisor_completion_cost
|
||||
|
||||
|
||||
def advisor_iterations_cost(usage: "Usage") -> tuple[float, float]:
|
||||
"""Prices each ``advisor_message`` iteration at the rate of the advisor model it names. Raises for an
|
||||
iteration without a model or whose model has no cost map entry."""
|
||||
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
|
||||
|
||||
def iteration_cost(iteration: Mapping[str, object]) -> tuple[float, float]:
|
||||
advisor_model: Final = iteration.get("model")
|
||||
if not isinstance(advisor_model, str) or not advisor_model:
|
||||
raise ValueError("Anthropic advisor_message iteration has no model, so it cannot be priced")
|
||||
litellm.get_model_info(model=advisor_model, custom_llm_provider="anthropic")
|
||||
cache_read: Final = int(iteration.get("cache_read_input_tokens") or 0)
|
||||
cache_write: Final = int(iteration.get("cache_creation_input_tokens") or 0)
|
||||
text_input: Final = int(iteration.get("input_tokens") or 0)
|
||||
return generic_cost_per_token(
|
||||
model=advisor_model,
|
||||
usage=Usage(
|
||||
prompt_tokens=text_input + cache_read + cache_write,
|
||||
completion_tokens=int(iteration.get("output_tokens") or 0),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
cached_tokens=cache_read, cache_creation_tokens=cache_write, text_tokens=text_input
|
||||
),
|
||||
cache_creation_input_tokens=cache_write,
|
||||
cache_read_input_tokens=cache_read,
|
||||
),
|
||||
custom_llm_provider="anthropic",
|
||||
)
|
||||
|
||||
costs: Final = tuple(
|
||||
iteration_cost(iteration)
|
||||
for iteration in getattr(usage, "iterations", None) or ()
|
||||
if isinstance(iteration, Mapping) and iteration.get("type") == "advisor_message"
|
||||
)
|
||||
return sum(prompt for prompt, _ in costs), sum(completion for _, completion in costs)
|
||||
|
||||
|
||||
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):
|
||||
|
|
|
|||
|
|
@ -4514,6 +4514,29 @@ async def enforce_key_and_fallback_model_access(
|
|||
user_model=None,
|
||||
)
|
||||
|
||||
for _advisor_model in dict.fromkeys(advisor_tool_models(request_data)):
|
||||
await can_key_call_model(
|
||||
model=_advisor_model,
|
||||
llm_model_list=llm_model_list,
|
||||
valid_token=valid_token,
|
||||
llm_router=llm_router,
|
||||
)
|
||||
|
||||
|
||||
def advisor_tool_models(request_data: Mapping[str, object]) -> list[str]:
|
||||
"""Models named by the request's ``advisor_20260301`` tools. A non-string model is returned as its repr so
|
||||
the allowlist check denies it instead of skipping it."""
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
|
||||
|
||||
tools: Final = request_data.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return []
|
||||
return [
|
||||
model if isinstance(model := tool.get("model"), str) else repr(model)
|
||||
for tool in tools
|
||||
if isinstance(tool, Mapping) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE
|
||||
]
|
||||
|
||||
|
||||
_enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias
|
||||
|
||||
|
|
|
|||
|
|
@ -2180,3 +2180,23 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
|
|||
assert response is not None
|
||||
assert response.choices[0].message.content == "Hello world"
|
||||
assert response.choices[0].message.reasoning_items == reasoning_items
|
||||
|
||||
|
||||
def test_chunk_builder_keeps_the_final_usage_iterations_for_cost_calculation():
|
||||
iterations: Final = [
|
||||
{"type": "message", "input_tokens": 412, "output_tokens": 89},
|
||||
{"type": "advisor_message", "model": "claude-opus-4-8", "input_tokens": 823, "output_tokens": 1612},
|
||||
]
|
||||
chunks: Final = [
|
||||
ModelResponseStream(choices=[], usage=Usage(prompt_tokens=412, completion_tokens=1, total_tokens=413)),
|
||||
ModelResponseStream(
|
||||
choices=[],
|
||||
usage=Usage(prompt_tokens=412, completion_tokens=89, total_tokens=501, iterations=iterations),
|
||||
),
|
||||
]
|
||||
|
||||
usage: Final = ChunkProcessor(chunks=chunks).calculate_usage(
|
||||
chunks=chunks, model="claude-haiku-4-5", completion_output=""
|
||||
)
|
||||
|
||||
assert usage.iterations == iterations
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag
|
|||
from litellm.llms.anthropic.chat import ModelResponseIterator
|
||||
from litellm.llms.anthropic.common_utils import process_anthropic_headers
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
|
||||
from litellm.exceptions import ModelNotMappedError
|
||||
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
|
||||
from litellm.llms.anthropic.pass_through.messages.transformation import (
|
||||
AnthropicMessagesConfig,
|
||||
|
|
@ -9000,6 +9002,67 @@ def test_calculate_usage_sums_cache_tokens_across_compaction_iterations():
|
|||
assert usage.prompt_tokens_details.cached_tokens == 17020
|
||||
|
||||
|
||||
ADVISOR_EXECUTOR_MODEL: Final = "claude-haiku-4-5"
|
||||
ADVISOR_MODEL: Final = "claude-opus-4-8"
|
||||
|
||||
|
||||
def _advisor_usage_object(advisor_model: object = ADVISOR_MODEL) -> dict[str, object]:
|
||||
return {
|
||||
"input_tokens": 1760,
|
||||
"cache_read_input_tokens": 412,
|
||||
"cache_creation_input_tokens": 0,
|
||||
"output_tokens": 531,
|
||||
"iterations": [
|
||||
{"type": "message", "input_tokens": 412, "output_tokens": 89},
|
||||
{"type": "advisor_message", "model": advisor_model, "input_tokens": 823, "output_tokens": 1612},
|
||||
{
|
||||
"type": "message",
|
||||
"input_tokens": 1348,
|
||||
"cache_read_input_tokens": 412,
|
||||
"output_tokens": 442,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_calculate_usage_keeps_advisor_iterations_out_of_the_request_model_tokens():
|
||||
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
|
||||
assert usage.prompt_tokens == 1760 + 412
|
||||
assert usage.completion_tokens == 531
|
||||
assert usage.prompt_tokens_details.cached_tokens == 412
|
||||
assert [iteration["type"] for iteration in usage.iterations] == ["message", "advisor_message", "message"]
|
||||
|
||||
|
||||
def test_cost_per_token_prices_advisor_iterations_at_the_advisor_models_rate():
|
||||
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
|
||||
executor: Final = litellm.get_model_info(model=ADVISOR_EXECUTOR_MODEL, custom_llm_provider="anthropic")
|
||||
advisor: Final = litellm.get_model_info(model=ADVISOR_MODEL, custom_llm_provider="anthropic")
|
||||
|
||||
prompt_cost, completion_cost = anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
|
||||
|
||||
executor_cost: Final = (
|
||||
1760 * executor["input_cost_per_token"]
|
||||
+ 412 * executor["cache_read_input_token_cost"]
|
||||
+ 531 * executor["output_cost_per_token"]
|
||||
)
|
||||
advisor_cost: Final = 823 * advisor["input_cost_per_token"] + 1612 * advisor["output_cost_per_token"]
|
||||
assert prompt_cost + completion_cost == pytest.approx(executor_cost + advisor_cost, rel=1e-9)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"advisor_model, expected_error",
|
||||
[("claude-no-such-advisor", ModelNotMappedError), (None, ValueError)],
|
||||
)
|
||||
def test_cost_per_token_rejects_an_advisor_iteration_it_cannot_price(
|
||||
advisor_model: object, expected_error: type[Exception]
|
||||
):
|
||||
usage: Final = AnthropicConfig().calculate_usage(
|
||||
usage_object=_advisor_usage_object(advisor_model), reasoning_content=None
|
||||
)
|
||||
with pytest.raises(expected_error):
|
||||
anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
|
||||
|
||||
|
||||
PROMPT_CACHING_MODEL: Final = "claude-sonnet-5-5"
|
||||
EPHEMERAL: Final = {"type": "ephemeral"}
|
||||
WEATHER_PARAMETERS: Final = {
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch
|
|||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy._types import LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.auth_utils import fallback_target_model_name, iter_request_fallback_targets
|
||||
from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access
|
||||
|
||||
|
|
@ -337,3 +337,37 @@ async def test_router_override_without_fallbacks_does_not_break_auth():
|
|||
)
|
||||
|
||||
assert seen == ["gpt-3.5-turbo"]
|
||||
|
||||
|
||||
def _advisor_request(advisor_model: str) -> dict:
|
||||
return {
|
||||
"model": "cheap-executor",
|
||||
"tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor_model}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_advisor_tool_model_outside_the_key_allowlist_is_denied():
|
||||
with pytest.raises(ProxyException) as denial:
|
||||
await enforce_key_and_fallback_model_access(
|
||||
valid_token=_key_with_models(["cheap-executor"]),
|
||||
request_data=_advisor_request("premium-advisor"),
|
||||
route="/v1/messages",
|
||||
request=None,
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
)
|
||||
|
||||
assert denial.value.type == ProxyErrorTypes.key_model_access_denied
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_advisor_tool_model_the_key_may_call_is_admitted():
|
||||
await enforce_key_and_fallback_model_access(
|
||||
valid_token=_key_with_models(["cheap-executor", "premium-advisor"]),
|
||||
request_data=_advisor_request("premium-advisor"),
|
||||
route="/v1/messages",
|
||||
request=None,
|
||||
llm_model_list=None,
|
||||
llm_router=None,
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue