diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index 68cdc204af4..28edfdbe071 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -1028,6 +1028,21 @@ class ChunkProcessor: ] return values[-1] if values else None + def _last_usage_iterations( + self, + chunks: Sequence["_UsageBearingChunk | ModelResponse"], + ) -> list | None: + """Last ``iterations`` list reported on a usage chunk (Anthropic sends it cumulatively on the final + ``message_delta``).""" + values: Final = [ + value + for chunk in chunks + if (usage_chunk := self._extract_usage_chunk(chunk)) is not None + and isinstance(value := getattr(usage_chunk, "iterations", None), list) + and len(value) > 0 + ] + return values[-1] if values else None + @staticmethod def _finish_reason_of_choice(choice: object) -> str | None: match choice: @@ -1203,6 +1218,9 @@ class ChunkProcessor: ) if value is not None } + usage_iterations: Final = self._last_usage_iterations(chunks) + if usage_iterations is not None: + provider_pricing_fields["iterations"] = usage_iterations returned_usage = Usage(**returned_usage.model_dump(), **provider_pricing_fields) diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 92e7c84031c..0b60091ec24 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -253,6 +253,8 @@ def _validated_defer_loading(value: object) -> bool: class _AnthropicUsageIteration(TypedDict, total=False): """One entry of the ``usage.iterations`` array on an Anthropic response.""" + type: ReadOnly[str] + model: ReadOnly[str] input_tokens: ReadOnly[int | None] output_tokens: ReadOnly[int | None] cache_creation_input_tokens: ReadOnly[int | None] @@ -2728,9 +2730,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ephemeral_1h_input_tokens=detailed_1h, ) + @staticmethod + def executor_billed_iterations(iterations: Sequence[Any]) -> list[Any]: + """Iterations billed at the request model's rates. ``advisor_message`` iterations ran on the advisor + model and are priced separately by ``advisor_iterations_cost``.""" + return [it for it in iterations if not (isinstance(it, Mapping) and it.get("type") == "advisor_message")] + @staticmethod def _resolve_cache_creation_token_details(usage: Mapping[str, Any]) -> CacheCreationTokenDetails | None: - iterations: Final = usage.get("iterations") + all_iterations: Final = usage.get("iterations") + iterations: Final = AnthropicConfig.executor_billed_iterations(all_iterations) if all_iterations else None if iterations: aggregated: Final = AnthropicConfig._aggregate_cache_creation_token_details(iterations) if aggregated is not None: @@ -2772,14 +2781,19 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): resolved_speed: Final = raw_speed if isinstance(raw_speed, str) else speed iterations: Final[Sequence[_AnthropicUsageIteration] | None] = _usage.get("iterations") - if iterations: - prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in iterations) - completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in iterations) - cache_creation_input_tokens = sum(it.get("cache_creation_input_tokens", 0) or 0 for it in iterations) - cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in iterations) + executor_iterations: Final[Sequence[_AnthropicUsageIteration] | None] = ( + self.executor_billed_iterations(iterations) if iterations else None + ) + if executor_iterations: + prompt_tokens = sum(it.get("input_tokens", 0) or 0 for it in executor_iterations) + completion_tokens = sum(it.get("output_tokens", 0) or 0 for it in executor_iterations) + cache_creation_input_tokens = sum( + it.get("cache_creation_input_tokens", 0) or 0 for it in executor_iterations + ) + cache_read_input_tokens = sum(it.get("cache_read_input_tokens", 0) or 0 for it in executor_iterations) prompt_tokens += cache_creation_input_tokens + cache_read_input_tokens - if not iterations: + if not executor_iterations: if "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None: cache_creation_input_tokens = _usage["cache_creation_input_tokens"] prompt_tokens += cache_creation_input_tokens @@ -2819,7 +2833,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) completion_token_details: Final = self._build_completion_token_details( usage_object=_usage, - iterations=iterations, + iterations=executor_iterations, completion_tokens=completion_tokens, reasoning_content=reasoning_content, completion_response=completion_response, diff --git a/litellm/llms/anthropic/cost_calculation.py b/litellm/llms/anthropic/cost_calculation.py index 94c8149eff3..14b0f2e46fc 100644 --- a/litellm/llms/anthropic/cost_calculation.py +++ b/litellm/llms/anthropic/cost_calculation.py @@ -3,6 +3,7 @@ Helper util for handling anthropic-specific cost calculation - e.g.: prompt caching """ +from collections.abc import Mapping from typing import TYPE_CHECKING, Final, Optional from pydantic import ValidationError @@ -64,7 +65,43 @@ def cost_per_token( except Exception: pass - return prompt_cost, completion_cost + advisor_prompt_cost, advisor_completion_cost = advisor_iterations_cost(usage) + return prompt_cost + advisor_prompt_cost, completion_cost + advisor_completion_cost + + +def advisor_iterations_cost(usage: "Usage") -> tuple[float, float]: + """Prices each ``advisor_message`` iteration at the rate of the advisor model it names. Raises for an + iteration without a model or whose model has no cost map entry.""" + from litellm.types.utils import PromptTokensDetailsWrapper, Usage + + def iteration_cost(iteration: Mapping[str, object]) -> tuple[float, float]: + advisor_model: Final = iteration.get("model") + if not isinstance(advisor_model, str) or not advisor_model: + raise ValueError("Anthropic advisor_message iteration has no model, so it cannot be priced") + litellm.get_model_info(model=advisor_model, custom_llm_provider="anthropic") + cache_read: Final = int(iteration.get("cache_read_input_tokens") or 0) + cache_write: Final = int(iteration.get("cache_creation_input_tokens") or 0) + text_input: Final = int(iteration.get("input_tokens") or 0) + return generic_cost_per_token( + model=advisor_model, + usage=Usage( + prompt_tokens=text_input + cache_read + cache_write, + completion_tokens=int(iteration.get("output_tokens") or 0), + prompt_tokens_details=PromptTokensDetailsWrapper( + cached_tokens=cache_read, cache_creation_tokens=cache_write, text_tokens=text_input + ), + cache_creation_input_tokens=cache_write, + cache_read_input_tokens=cache_read, + ), + custom_llm_provider="anthropic", + ) + + costs: Final = tuple( + iteration_cost(iteration) + for iteration in getattr(usage, "iterations", None) or () + if isinstance(iteration, Mapping) and iteration.get("type") == "advisor_message" + ) + return sum(prompt for prompt, _ in costs), sum(completion for _, completion in costs) class _AnthropicServerToolUseProbe(LiteLLMBaseModel): diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index b1e8bdbe787..90d233b5a54 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -4546,6 +4546,29 @@ async def enforce_key_and_fallback_model_access( user_model=None, ) + for _advisor_model in dict.fromkeys(advisor_tool_models(request_data)): + await can_key_call_model( + model=_advisor_model, + llm_model_list=llm_model_list, + valid_token=valid_token, + llm_router=llm_router, + ) + + +def advisor_tool_models(request_data: Mapping[str, object]) -> list[str]: + """Models named by the request's ``advisor_20260301`` tools. A non-string model is returned as its repr so + the allowlist check denies it instead of skipping it.""" + from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE + + tools: Final = request_data.get("tools") + if not isinstance(tools, list): + return [] + return [ + model if isinstance(model := tool.get("model"), str) else repr(model) + for tool in tools + if isinstance(tool, Mapping) and tool.get("type") == ANTHROPIC_ADVISOR_TOOL_TYPE + ] + _enforce_key_and_fallback_model_access: Final = enforce_key_and_fallback_model_access # pyright: ignore[reportUnknownVariableType] # existing tests use this alias diff --git a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py index 5c56245cf33..4539444f1c2 100644 --- a/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/unit/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -2517,3 +2517,23 @@ def test_stream_chunk_builder_keeps_reasoning_items_on_text_only_replies(): assert response is not None assert response.choices[0].message.content == "Hello world" assert response.choices[0].message.reasoning_items == reasoning_items + + +def test_chunk_builder_keeps_the_final_usage_iterations_for_cost_calculation(): + iterations: Final = [ + {"type": "message", "input_tokens": 412, "output_tokens": 89}, + {"type": "advisor_message", "model": "claude-opus-4-8", "input_tokens": 823, "output_tokens": 1612}, + ] + chunks: Final = [ + ModelResponseStream(choices=[], usage=Usage(prompt_tokens=412, completion_tokens=1, total_tokens=413)), + ModelResponseStream( + choices=[], + usage=Usage(prompt_tokens=412, completion_tokens=89, total_tokens=501, iterations=iterations), + ), + ] + + usage: Final = ChunkProcessor(chunks=chunks).calculate_usage( + chunks=chunks, model="claude-haiku-4-5", completion_output="" + ) + + assert usage.iterations == iterations diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index b3b04440d16..ed719e71831 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -25,6 +25,8 @@ from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messag from litellm.llms.anthropic.chat import ModelResponseIterator from litellm.llms.anthropic.common_utils import process_anthropic_headers from litellm.llms.anthropic.chat.transformation import AnthropicConfig +from litellm.llms.anthropic.cost_calculation import cost_per_token as anthropic_cost_per_token +from litellm.exceptions import ModelNotMappedError from litellm.llms.base_llm.base_utils import ResponseFormatSchema, response_format_schema from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -9077,6 +9079,67 @@ def test_calculate_usage_sums_cache_tokens_across_compaction_iterations(): assert usage.prompt_tokens_details.cached_tokens == 17020 +ADVISOR_EXECUTOR_MODEL: Final = "claude-haiku-4-5" +ADVISOR_MODEL: Final = "claude-opus-4-8" + + +def _advisor_usage_object(advisor_model: object = ADVISOR_MODEL) -> dict[str, object]: + return { + "input_tokens": 1760, + "cache_read_input_tokens": 412, + "cache_creation_input_tokens": 0, + "output_tokens": 531, + "iterations": [ + {"type": "message", "input_tokens": 412, "output_tokens": 89}, + {"type": "advisor_message", "model": advisor_model, "input_tokens": 823, "output_tokens": 1612}, + { + "type": "message", + "input_tokens": 1348, + "cache_read_input_tokens": 412, + "output_tokens": 442, + }, + ], + } + + +def test_calculate_usage_keeps_advisor_iterations_out_of_the_request_model_tokens(): + usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None) + assert usage.prompt_tokens == 1760 + 412 + assert usage.completion_tokens == 531 + assert usage.prompt_tokens_details.cached_tokens == 412 + assert [iteration["type"] for iteration in usage.iterations] == ["message", "advisor_message", "message"] + + +def test_cost_per_token_prices_advisor_iterations_at_the_advisor_models_rate(): + usage: Final = AnthropicConfig().calculate_usage(usage_object=_advisor_usage_object(), reasoning_content=None) + executor: Final = litellm.get_model_info(model=ADVISOR_EXECUTOR_MODEL, custom_llm_provider="anthropic") + advisor: Final = litellm.get_model_info(model=ADVISOR_MODEL, custom_llm_provider="anthropic") + + prompt_cost, completion_cost = anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage) + + executor_cost: Final = ( + 1760 * executor["input_cost_per_token"] + + 412 * executor["cache_read_input_token_cost"] + + 531 * executor["output_cost_per_token"] + ) + advisor_cost: Final = 823 * advisor["input_cost_per_token"] + 1612 * advisor["output_cost_per_token"] + assert prompt_cost + completion_cost == pytest.approx(executor_cost + advisor_cost, rel=1e-9) + + +@pytest.mark.parametrize( + "advisor_model, expected_error", + [("claude-no-such-advisor", ModelNotMappedError), (None, ValueError)], +) +def test_cost_per_token_rejects_an_advisor_iteration_it_cannot_price( + advisor_model: object, expected_error: type[Exception] +): + usage: Final = AnthropicConfig().calculate_usage( + usage_object=_advisor_usage_object(advisor_model), reasoning_content=None + ) + with pytest.raises(expected_error): + anthropic_cost_per_token(model=ADVISOR_EXECUTOR_MODEL, usage=usage) + + PROMPT_CACHING_MODEL: Final = "claude-sonnet-5-5" EPHEMERAL: Final = {"type": "ephemeral"} WEATHER_PARAMETERS: Final = { diff --git a/tests/unit/proxy/auth/test_router_override_fallback_auth.py b/tests/unit/proxy/auth/test_router_override_fallback_auth.py index d35a5851659..614d63c4dd4 100644 --- a/tests/unit/proxy/auth/test_router_override_fallback_auth.py +++ b/tests/unit/proxy/auth/test_router_override_fallback_auth.py @@ -10,7 +10,7 @@ from unittest.mock import AsyncMock, patch import pytest -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, ProxyErrorTypes, ProxyException, UserAPIKeyAuth from litellm.proxy.auth.auth_utils import fallback_target_model_name, iter_request_fallback_targets from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access @@ -337,3 +337,37 @@ async def test_router_override_without_fallbacks_does_not_break_auth(): ) assert seen == ["gpt-3.5-turbo"] + + +def _advisor_request(advisor_model: str) -> dict: + return { + "model": "cheap-executor", + "tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor_model}], + } + + +@pytest.mark.asyncio +async def test_advisor_tool_model_outside_the_key_allowlist_is_denied(): + with pytest.raises(ProxyException) as denial: + await enforce_key_and_fallback_model_access( + valid_token=_key_with_models(["cheap-executor"]), + request_data=_advisor_request("premium-advisor"), + route="/v1/messages", + request=None, + llm_model_list=None, + llm_router=None, + ) + + assert denial.value.type == ProxyErrorTypes.key_model_access_denied + + +@pytest.mark.asyncio +async def test_advisor_tool_model_the_key_may_call_is_admitted(): + await enforce_key_and_fallback_model_access( + valid_token=_key_with_models(["cheap-executor", "premium-advisor"]), + request_data=_advisor_request("premium-advisor"), + route="/v1/messages", + request=None, + llm_model_list=None, + llm_router=None, + )