From 9f894486c9411c1bc807152de2aafb9ecf600063 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:08:10 -0700 Subject: [PATCH] fix(vertex_ai): bill token-priced Claude deployments at the served model's search price A Vertex Claude deployment priced in model_info without a search price now bills each web search at the served model's cost map price, read at cost time and never written into the deployment entry. An explicit search price, zero included, still wins. --- .../llm_cost_calc/tool_call_cost_tracking.py | 1 + litellm/llms/__init__.py | 27 ++++++- .../cost_calculation/cases/vertex_ai.py | 17 ++++- .../test_tool_call_cost_tracking.py | 70 ++++++++++++++++--- 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 2a46b8a2c37..54213f40ac0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -215,6 +215,7 @@ class StandardBuiltInToolCostTracking: usage=resolved_usage, model_info=model_info, response_model=response_model if isinstance(response_model, str) else None, + lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info, ) if result is not None: return result diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index d8db55dbfd4..95a9e689290 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from collections.abc import Callable + from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, ) @@ -30,8 +32,25 @@ def get_cost_for_google_maps_grounding_request( return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) +def _with_served_model_search_pricing( + model_info: "ModelInfo", + response_model: str | None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", +) -> "ModelInfo": + if model_info.get("search_context_cost_per_query") is not None or response_model is None: + return model_info + served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None + if served_model_info is None: + return model_info + return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} + + def get_cost_for_web_search_request( - custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo", response_model: str | None = None + custom_llm_provider: str, + usage: "Usage", + model_info: "ModelInfo", + response_model: str | None = None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None, ) -> float | None: """ Get the cost for a web search request for a given model. @@ -41,6 +60,7 @@ def get_cost_for_web_search_request( usage: The usage object. model_info: The model info. response_model: The model the provider response names. + lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped. """ if custom_llm_provider == "gemini": from .gemini.cost_calculator import cost_per_web_search_request @@ -59,7 +79,10 @@ def get_cost_for_web_search_request( from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info), + usage=usage, + ) from .vertex_ai.gemini.cost_calculator import ( cost_per_web_search_request as cost_per_web_search_request_vertex_ai, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 8991de6cdd6..8590a19646b 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3388,8 +3388,6 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( ) - -# vertex_ai/claude-sonnet-5-5 deployment priced in its model_info, so it bills under the deployment's own id VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment( input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01 ) @@ -3517,6 +3515,20 @@ VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = ( Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0), ) ) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.034, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06), + ) +) VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests", "/v1/chat/completions", @@ -3943,6 +3955,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM, VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 09c5c34a91d..678776aa5e7 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1053,23 +1053,27 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None -def _register_priced_vertex_deployment(model: str, search_cost_per_query: float) -> str: +def _register_priced_vertex_deployment(model: str, search_cost_per_query: float | None) -> str: from litellm import Router + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + search_rates: Final = ( + {} + if search_cost_per_query is None + else { + "search_context_cost_per_query": { + "search_context_size_low": search_cost_per_query, + "search_context_size_medium": search_cost_per_query, + "search_context_size_high": search_cost_per_query, + } + } + ) router: Final = Router( model_list=[ { "model_name": "priced-vertex-deployment", "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"}, - "model_info": { - "input_cost_per_token": 3e-6, - "output_cost_per_token": 1.5e-5, - "search_context_cost_per_query": { - "search_context_size_low": search_cost_per_query, - "search_context_size_medium": search_cost_per_query, - "search_context_size_high": search_cost_per_query, - }, - }, + "model_info": {**token_rates, **search_rates}, } ] ) @@ -1123,6 +1127,52 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request( assert cost == pytest.approx(web_search_requests * search_cost_per_query) +@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response]) +def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price( + local_model_cost_map: None, + build_response: Callable[[int], tuple[object, Usage | None]], +) -> None: + served_search_pricing: Final = ( + litellm.get_model_info(model="vertex_ai/claude-sonnet-5-5").get("search_context_cost_per_query") or {} + ) + served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 + deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None) + response_object, usage = build_response(2) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=response_object, + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert served_search_price > 0 + assert cost == pytest.approx(2 * served_search_price) + + +def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( + local_model_cost_map: None, +) -> None: + deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=ModelResponse(model="claude-unmapped-preview", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None: search_cost_per_prompt: Final = 0.03 deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)