fix(cost): bill served-model search price on token-only Anthropic and Azure AI Claude deployments

Token-only Claude deployments on anthropic and azure_ai billed web search at 0 because the deployment entry has no search price. They now fall back to the served model's cost map search price, as vertex_ai already does. Vertex routing reads the served model first, so a Gemini deployment whose id contains claude keeps per-prompt grounding
This commit is contained in:
mateo-berri 2026-10-30 05:45:31 -07:00
parent 217fc1bc1b
commit 1ca8c8c94a
2 changed files with 44 additions and 25 deletions

View file

@ -35,11 +35,14 @@ def get_cost_for_google_maps_grounding_request(
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
served_model_provider: str,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None
served_model_info: Final = (
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
)
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
@ -69,18 +72,23 @@ def get_cost_for_web_search_request(
elif custom_llm_provider == "anthropic":
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
usage=usage,
)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: Final[str] = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower() or "claude" in (response_model or "").lower():
if "claude" in (response_model or model_key).lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info),
model_info=_with_served_model_search_pricing(
model_info, response_model, "vertex_ai", lookup_model_info
),
usage=usage,
)
@ -97,7 +105,10 @@ def get_cost_for_web_search_request(
return None
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
usage=usage,
)
elif custom_llm_provider == "perplexity":
# Perplexity handles search costs internally in its own cost calculator
# Return 0.0 to indicate costs are already accounted for

View file

@ -1053,7 +1053,7 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
def _register_priced_vertex_deployment(
def _register_priced_deployment(
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
) -> str:
from litellm import Router
@ -1073,8 +1073,8 @@ def _register_priced_vertex_deployment(
router: Final = Router(
model_list=[
{
"model_name": "priced-vertex-deployment",
"litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"},
"model_name": "priced-deployment",
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
}
]
@ -1082,7 +1082,7 @@ def _register_priced_vertex_deployment(
return router.model_list[0]["model_info"]["id"]
def _vertex_claude_chat_response(
def _claude_chat_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
usage: Final = Usage(
@ -1094,7 +1094,7 @@ def _vertex_claude_chat_response(
return ModelResponse(model=model, usage=usage), usage
def _vertex_claude_messages_response(
def _claude_messages_response(
web_search_requests: int, model: str = "claude-sonnet-5-5"
) -> tuple[object, Usage | None]:
return {
@ -1111,7 +1111,7 @@ def _vertex_claude_messages_response(
}, None
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
local_model_cost_map: None,
@ -1119,7 +1119,7 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
web_search_requests: int,
search_cost_per_query: float,
) -> None:
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
response_object, usage = build_response(web_search_requests)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
@ -1133,37 +1133,40 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
@pytest.mark.parametrize(
("deployment_model", "served_model"),
("deployment_model", "served_model", "custom_llm_provider"),
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
[
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5"),
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929"),
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001"),
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
],
)
@pytest.mark.parametrize("deployment_id", [None, "vertex-claude-token-priced"], ids=["hashed_id", "claude_in_id"])
def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price(
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
local_model_cost_map: None,
build_response: Callable[[int, str], tuple[object, Usage | None]],
deployment_model: str,
served_model: str,
custom_llm_provider: str,
deployment_id: str | None,
) -> None:
served_search_pricing: Final = (
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
)
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
registered_id: Final = _register_priced_vertex_deployment(deployment_model, None, deployment_id)
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
response_object, usage = build_response(2, served_model)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=registered_id,
response_object=response_object,
usage=usage,
custom_llm_provider="vertex_ai",
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=None,
)
@ -1174,7 +1177,7 @@ def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_searc
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
local_model_cost_map: None,
) -> None:
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
@ -1193,9 +1196,14 @@ def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_mode
assert cost == 0.0
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None:
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
local_model_cost_map: None, deployment_id: str | None
) -> None:
search_cost_per_prompt: Final = 0.03
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)
registered_id: Final = _register_priced_deployment(
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
)
usage: Final = Usage(
prompt_tokens=1840,
completion_tokens=412,
@ -1204,7 +1212,7 @@ def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model
)
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=deployment_id,
model=registered_id,
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
usage=usage,
custom_llm_provider="vertex_ai",