diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 2a46b8a2c37..54213f40ac0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -215,6 +215,7 @@ class StandardBuiltInToolCostTracking: usage=resolved_usage, model_info=model_info, response_model=response_model if isinstance(response_model, str) else None, + lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info, ) if result is not None: return result diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index d8db55dbfd4..95a9e689290 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from collections.abc import Callable + from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, ) @@ -30,8 +32,25 @@ def get_cost_for_google_maps_grounding_request( return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) +def _with_served_model_search_pricing( + model_info: "ModelInfo", + response_model: str | None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", +) -> "ModelInfo": + if model_info.get("search_context_cost_per_query") is not None or response_model is None: + return model_info + served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None + if served_model_info is None: + return model_info + return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} + + def get_cost_for_web_search_request( - custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo", response_model: str | None = None + custom_llm_provider: str, + usage: "Usage", + model_info: "ModelInfo", + response_model: str | None = None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None, ) -> float | None: """ Get the cost for a web search request for a given model. @@ -41,6 +60,7 @@ def get_cost_for_web_search_request( usage: The usage object. model_info: The model info. response_model: The model the provider response names. + lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped. """ if custom_llm_provider == "gemini": from .gemini.cost_calculator import cost_per_web_search_request @@ -59,7 +79,10 @@ def get_cost_for_web_search_request( from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info), + usage=usage, + ) from .vertex_ai.gemini.cost_calculator import ( cost_per_web_search_request as cost_per_web_search_request_vertex_ai, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 8991de6cdd6..8590a19646b 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3388,8 +3388,6 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( ) - -# vertex_ai/claude-sonnet-5-5 deployment priced in its model_info, so it bills under the deployment's own id VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment( input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01 ) @@ -3517,6 +3515,20 @@ VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = ( Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0), ) ) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.034, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06), + ) +) VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests", "/v1/chat/completions", @@ -3943,6 +3955,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 09c5c34a91d..678776aa5e7 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1053,23 +1053,27 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None -def _register_priced_vertex_deployment(model: str, search_cost_per_query: float) -> str: +def _register_priced_vertex_deployment(model: str, search_cost_per_query: float | None) -> str: from litellm import Router + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + search_rates: Final = ( + {} + if search_cost_per_query is None + else { + "search_context_cost_per_query": { + "search_context_size_low": search_cost_per_query, + "search_context_size_medium": search_cost_per_query, + "search_context_size_high": search_cost_per_query, + } + } + ) router: Final = Router( model_list=[ { "model_name": "priced-vertex-deployment", "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"}, - "model_info": { - "input_cost_per_token": 3e-6, - "output_cost_per_token": 1.5e-5, - "search_context_cost_per_query": { - "search_context_size_low": search_cost_per_query, - "search_context_size_medium": search_cost_per_query, - "search_context_size_high": search_cost_per_query, - }, - }, + "model_info": {**token_rates, **search_rates}, } ] ) @@ -1123,6 +1127,52 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request( assert cost == pytest.approx(web_search_requests * search_cost_per_query) +@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response]) +def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price( + local_model_cost_map: None, + build_response: Callable[[int], tuple[object, Usage | None]], +) -> None: + served_search_pricing: Final = ( + litellm.get_model_info(model="vertex_ai/claude-sonnet-5-5").get("search_context_cost_per_query") or {} + ) + served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 + deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None) + response_object, usage = build_response(2) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=response_object, + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert served_search_price > 0 + assert cost == pytest.approx(2 * served_search_price) + + +def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( + local_model_cost_map: None, +) -> None: + deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=ModelResponse(model="claude-unmapped-preview", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None: search_cost_per_prompt: Final = 0.03 deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)