fix(vertex_ai): bill token-priced Claude deployments at the served model's search price

A Vertex Claude deployment priced in model_info without a search price now bills each web search at the served model's cost map price, read at cost time and never written into the deployment entry. An explicit search price, zero included, still wins.
This commit is contained in:
mateo-berri 2026-10-29 20:08:10 -07:00
parent 112ed5c534
commit 9f894486c9
4 changed files with 101 additions and 14 deletions

View file

@ -215,6 +215,7 @@ class StandardBuiltInToolCostTracking:
usage=resolved_usage,
model_info=model_info,
response_model=response_model if isinstance(response_model, str) else None,
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
)
if result is not None:
return result

View file

@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
from . import *
if TYPE_CHECKING:
from collections.abc import Callable
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
)
@ -30,8 +32,25 @@ def get_cost_for_google_maps_grounding_request(
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
def get_cost_for_web_search_request(
custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo", response_model: str | None = None
custom_llm_provider: str,
usage: "Usage",
model_info: "ModelInfo",
response_model: str | None = None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
) -> float | None:
"""
Get the cost for a web search request for a given model.
@ -41,6 +60,7 @@ def get_cost_for_web_search_request(
usage: The usage object.
model_info: The model info.
response_model: The model the provider response names.
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
"""
if custom_llm_provider == "gemini":
from .gemini.cost_calculator import cost_per_web_search_request
@ -59,7 +79,10 @@ def get_cost_for_web_search_request(
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info),
usage=usage,
)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,

View file

@ -3388,8 +3388,6 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
)
# vertex_ai/claude-sonnet-5-5 deployment priced in its model_info, so it bills under the deployment's own id
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
)
@ -3517,6 +3515,20 @@ VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.034,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
"/v1/chat/completions",
@ -3943,6 +3955,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,

View file

@ -1053,23 +1053,27 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
def _register_priced_vertex_deployment(model: str, search_cost_per_query: float) -> str:
def _register_priced_vertex_deployment(model: str, search_cost_per_query: float | None) -> str:
from litellm import Router
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
search_rates: Final = (
{}
if search_cost_per_query is None
else {
"search_context_cost_per_query": {
"search_context_size_low": search_cost_per_query,
"search_context_size_medium": search_cost_per_query,
"search_context_size_high": search_cost_per_query,
}
}
)
router: Final = Router(
model_list=[
{
"model_name": "priced-vertex-deployment",
"litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"},
"model_info": {
"input_cost_per_token": 3e-6,
"output_cost_per_token": 1.5e-5,
"search_context_cost_per_query": {
"search_context_size_low": search_cost_per_query,
"search_context_size_medium": search_cost_per_query,
"search_context_size_high": search_cost_per_query,
},
},
"model_info": {**token_rates, **search_rates},
}
]
)
@ -1123,6 +1127,52 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price(
local_model_cost_map: None,
build_response: Callable[[int], tuple[object, Usage | None]],
) -> None:
served_search_pricing: Final = (
litellm.get_model_info(model="vertex_ai/claude-sonnet-5-5").get("search_context_cost_per_query") or {}
)
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
response_object, usage = build_response(2)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=response_object,
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert served_search_price > 0
assert cost == pytest.approx(2 * served_search_price)
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
local_model_cost_map: None,
) -> None:
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
total_tokens=2252,
server_tool_use=ServerToolUse(web_search_requests=2),
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",
standard_built_in_tools_params=None,
)
assert cost == 0.0
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None:
search_cost_per_prompt: Final = 0.03
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)