This commit is contained in:
tomatomptt 2026-10-27 02:21:28 +00:00 • committed by GitHub
commit 7253739c0f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 219 additions and 10 deletions

View file

@ -1028,6 +1028,21 @@ class ChunkProcessor:
]
return values[-1] if values else None
def _last_usage_iterations(
self,
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
) -> list | None:
"""Last ``iterations`` list reported on a usage chunk (Anthropic sends it cumulatively on the final
``message_delta``)."""
values: Final = [
value
for chunk in chunks
if (usage_chunk := self._extract_usage_chunk(chunk)) is not None
and isinstance(value := getattr(usage_chunk, "iterations", None), list)
and len(value) > 0
]
return values[-1] if values else None
@staticmethod
def _finish_reason_of_choice(choice: object) -> str | None:
match choice:
@ -1203,6 +1218,9 @@ class ChunkProcessor:
)
if value is not None
}
usage_iterations: Final = self._last_usage_iterations(chunks)
if usage_iterations is not None:
provider_pricing_fields["iterations"] = usage_iterations
returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields)

View file

@ -253,6 +253,8 @@ def _validated_defer_loading(value: object) -> bool:
class _AnthropicUsageIteration(TypedDict, total=False):
"""One entry of the ``usage.iterations`` array on an Anthropic response."""
type: ReadOnly[str]
model: ReadOnly[str]
input_tokens: ReadOnly[int | None]
output_tokens: ReadOnly[int | None]
cache_creation_input_tokens: ReadOnly[int | None]
@ -2728,9 +2730,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
ephemeral_1h_input_tokens=detailed_1h,
)
@staticmethod
def executor_billed_iterations(iterations: Sequence[Any]) -> list[Any]:
"""Iterations billed at the request model's rates. ``advisor_message`` iterations ran on the advisor
model and are priced separately by ``advisor_iterations_cost``."""
return [it for it in iterations if not (isinstance(it, Mapping) and it.get("type") == "advisor_message")]
@staticmethod
def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None:
iterations: Final = usage.get("iterations")
all_iterations: Final = usage.get("iterations")
iterations: Final = AnthropicConfig.executor_billed_iterations(all_iterations) if all_iterations else None
if iterations:
aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations)
if aggregated is not None:
@ -2772,14 +2781,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed
iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations")
if iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations)
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations)
cache_creation_input_tokens = sum(it.get("cache_creation_input_tokens", 0) or 0 for it in iterations)
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in iterations)
executor_iterations: Final[Sequence[_AnthropicUsageIteration] | None] = (
self.executor_billed_iterations(iterations) if iterations else None
)
if executor_iterations:
prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in executor_iterations)
completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in executor_iterations)
cache_creation_input_tokens = sum(
it.get("cache_creation_input_tokens", 0) or 0 for it in executor_iterations
)
cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in executor_iterations)
prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens
if not iterations:
if not executor_iterations:
if "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None:
cache_creation_input_tokens = _usage["cache_creation_input_tokens"]
prompt_tokens += cache_creation_input_tokens
@ -2819,7 +2833,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
)
completion_token_details: Final = self._build_completion_token_details(
usage_object=_usage,
iterations=iterations,
iterations=executor_iterations,
completion_tokens=completion_tokens,
reasoning_content=reasoning_content,
completion_response=completion_response,

View file

@ -3,6 +3,7 @@ Helper util for handling anthropic-specific cost calculation
- e.g.: prompt caching
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Optional
from pydantic import ValidationError
@ -64,7 +65,43 @@ def cost_per_token(
except Exception:
pass
return prompt_cost, completion_cost
advisor_prompt_cost, advisor_completion_cost = advisor_iterations_cost(usage)
return prompt_cost + advisor_prompt_cost, completion_cost + advisor_completion_cost
def advisor_iterations_cost(usage: "Usage") -> tuple[float, float]:
"""Prices each ``advisor_message`` iteration at the rate of the advisor model it names. Raises for an
iteration without a model or whose model has no cost map entry."""
from litellm.types.utils import PromptTokensDetailsWrapper, Usage
def iteration_cost(iteration: Mapping[str, object]) -> tuple[float, float]:
advisor_model: Final = iteration.get("model")
if not isinstance(advisor_model, str) or not advisor_model:
raise ValueError("Anthropic advisor_message iteration has no model, so it cannot be priced")
litellm.get_model_info(model=advisor_model, custom_llm_provider="anthropic")
cache_read: Final = int(iteration.get("cache_read_input_tokens") or 0)
cache_write: Final = int(iteration.get("cache_creation_input_tokens") or 0)
text_input: Final = int(iteration.get("input_tokens") or 0)
return generic_cost_per_token(
model=advisor_model,
usage=Usage(
prompt_tokens=text_input + cache_read + cache_write,
completion_tokens=int(iteration.get("output_tokens") or 0),
prompt_tokens_details=PromptTokensDetailsWrapper(
cached_tokens=cache_read, cache_creation_tokens=cache_write, text_tokens=text_input
),
cache_creation_input_tokens=cache_write,
cache_read_input_tokens=cache_read,
),
custom_llm_provider="anthropic",
)
costs: Final = tuple(
iteration_cost(iteration)
for iteration in getattr(usage, "iterations", None) or ()
if isinstance(iteration, Mapping) and iteration.get("type") == "advisor_message"
)
return sum(prompt for prompt, _ in costs), sum(completion for _, completion in costs)
class _AnthropicServerToolUseProbe(LiteLLMBaseModel):

View file

@ -4546,6 +4546,29 @@ async def enforce_key_and_fallback_model_access(
user_model=None,
)
for _advisor_model in dict.fromkeys(advisor_tool_models(request_data)):
await can_key_call_model(
model=_advisor_model,
llm_model_list=llm_model_list,
valid_token=valid_token,
llm_router=llm_router,
)
def advisor_tool_models(request_data: Mapping[str, object]) -> list[str]:
"""Models named by the request's ``advisor_20260301`` tools. A non-string model is returned as its repr so
the allowlist check denies it instead of skipping it."""
from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE
tools: Final = request_data.get("tools")
if not isinstance(tools, list):
return []
return [
model if isinstance(model := tool.get("model"), str) else repr(model)
for tool in tools
if isinstance(tool, Mapping) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE
]
_enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias

View file

@ -2517,3 +2517,23 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies():
assert response is not None
assert response.choices[0].message.content == "Hello world"
assert response.choices[0].message.reasoning_items == reasoning_items
def test_chunk_builder_keeps_the_final_usage_iterations_for_cost_calculation():
iterations: Final = [
{"type": "message", "input_tokens": 412, "output_tokens": 89},
{"type": "advisor_message", "model": "claude-opus-4-8", "input_tokens": 823, "output_tokens": 1612},
]
chunks: Final = [
ModelResponseStream(choices=[], usage=Usage(prompt_tokens=412, completion_tokens=1, total_tokens=413)),
ModelResponseStream(
choices=[],
usage=Usage(prompt_tokens=412, completion_tokens=89, total_tokens=501, iterations=iterations),
),
]
usage: Final = ChunkProcessor(chunks=chunks).calculate_usage(
chunks=chunks, model="claude-haiku-4-5", completion_output=""
)
assert usage.iterations == iterations

View file

@ -25,6 +25,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag
from litellm.llms.anthropic.chat import ModelResponseIterator
from litellm.llms.anthropic.common_utils import process_anthropic_headers
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token
from litellm.exceptions import ModelNotMappedError
from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
@ -9077,6 +9079,67 @@ def test_calculate_usage_sums_cache_tokens_across_compaction_iterations():
assert usage.prompt_tokens_details.cached_tokens == 17020
ADVISOR_EXECUTOR_MODEL: Final = "claude-haiku-4-5"
ADVISOR_MODEL: Final = "claude-opus-4-8"
def _advisor_usage_object(advisor_model: object = ADVISOR_MODEL) -> dict[str, object]:
return {
"input_tokens": 1760,
"cache_read_input_tokens": 412,
"cache_creation_input_tokens": 0,
"output_tokens": 531,
"iterations": [
{"type": "message", "input_tokens": 412, "output_tokens": 89},
{"type": "advisor_message", "model": advisor_model, "input_tokens": 823, "output_tokens": 1612},
{
"type": "message",
"input_tokens": 1348,
"cache_read_input_tokens": 412,
"output_tokens": 442,
},
],
}
def test_calculate_usage_keeps_advisor_iterations_out_of_the_request_model_tokens():
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
assert usage.prompt_tokens == 1760 + 412
assert usage.completion_tokens == 531
assert usage.prompt_tokens_details.cached_tokens == 412
assert [iteration["type"] for iteration in usage.iterations] == ["message", "advisor_message", "message"]
def test_cost_per_token_prices_advisor_iterations_at_the_advisor_models_rate():
usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None)
executor: Final = litellm.get_model_info(model=ADVISOR_EXECUTOR_MODEL, custom_llm_provider="anthropic")
advisor: Final = litellm.get_model_info(model=ADVISOR_MODEL, custom_llm_provider="anthropic")
prompt_cost, completion_cost = anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
executor_cost: Final = (
1760 * executor["input_cost_per_token"]
+ 412 * executor["cache_read_input_token_cost"]
+ 531 * executor["output_cost_per_token"]
)
advisor_cost: Final = 823 * advisor["input_cost_per_token"] + 1612 * advisor["output_cost_per_token"]
assert prompt_cost + completion_cost == pytest.approx(executor_cost + advisor_cost, rel=1e-9)
@pytest.mark.parametrize(
"advisor_model, expected_error",
[("claude-no-such-advisor", ModelNotMappedError), (None, ValueError)],
)
def test_cost_per_token_rejects_an_advisor_iteration_it_cannot_price(
advisor_model: object, expected_error: type[Exception]
):
usage: Final = AnthropicConfig().calculate_usage(
usage_object=_advisor_usage_object(advisor_model), reasoning_content=None
)
with pytest.raises(expected_error):
anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage)
PROMPT_CACHING_MODEL: Final = "claude-sonnet-5-5"
EPHEMERAL: Final = {"type": "ephemeral"}
WEATHER_PARAMETERS: Final = {

View file

@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch
import pytest
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy._types import LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import fallback_target_model_name, iter_request_fallback_targets
from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access
@ -337,3 +337,37 @@ async def test_router_override_without_fallbacks_does_not_break_auth():
)
assert seen == ["gpt-3.5-turbo"]
def _advisor_request(advisor_model: str) -> dict:
return {
"model": "cheap-executor",
"tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor_model}],
}
@pytest.mark.asyncio
async def test_advisor_tool_model_outside_the_key_allowlist_is_denied():
with pytest.raises(ProxyException) as denial:
await enforce_key_and_fallback_model_access(
valid_token=_key_with_models(["cheap-executor"]),
request_data=_advisor_request("premium-advisor"),
route="/v1/messages",
request=None,
llm_model_list=None,
llm_router=None,
)
assert denial.value.type == ProxyErrorTypes.key_model_access_denied
@pytest.mark.asyncio
async def test_advisor_tool_model_the_key_may_call_is_admitted():
await enforce_key_and_fallback_model_access(
valid_token=_key_with_models(["cheap-executor", "premium-advisor"]),
request_data=_advisor_request("premium-advisor"),
route="/v1/messages",
request=None,
llm_model_list=None,
llm_router=None,
)