mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(cost): bill served-model search price on token-only Anthropic and Azure AI Claude deployments
Token-only Claude deployments on anthropic and azure_ai billed web search at 0 because the deployment entry has no search price. They now fall back to the served model's cost map search price, as vertex_ai already does. Vertex routing reads the served model first, so a Gemini deployment whose id contains claude keeps per-prompt grounding
This commit is contained in:
parent
217fc1bc1b
commit
1ca8c8c94a
2 changed files with 44 additions and 25 deletions
|
|
@ -35,11 +35,14 @@ def get_cost_for_google_maps_grounding_request(
|
|||
def _with_served_model_search_pricing(
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None,
|
||||
served_model_provider: str,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
|
||||
) -> "ModelInfo":
|
||||
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
|
||||
return model_info
|
||||
served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None
|
||||
served_model_info: Final = (
|
||||
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
|
||||
)
|
||||
if served_model_info is None:
|
||||
return model_info
|
||||
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
|
||||
|
|
@ -69,18 +72,23 @@ def get_cost_for_web_search_request(
|
|||
elif custom_llm_provider == "anthropic":
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider.startswith("vertex_ai"):
|
||||
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
|
||||
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
|
||||
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
|
||||
model_key: Final[str] = model_info.get("key", "") if model_info else ""
|
||||
if "claude" in model_key.lower() or "claude" in (response_model or "").lower():
|
||||
if "claude" in (response_model or model_key).lower():
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info),
|
||||
model_info=_with_served_model_search_pricing(
|
||||
model_info, response_model, "vertex_ai", lookup_model_info
|
||||
),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
|
|
@ -97,7 +105,10 @@ def get_cost_for_web_search_request(
|
|||
return None
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
# Perplexity handles search costs internally in its own cost calculator
|
||||
# Return 0.0 to indicate costs are already accounted for
|
||||
|
|
|
|||
|
|
@ -1053,7 +1053,7 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
|
|||
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
|
||||
|
||||
|
||||
def _register_priced_vertex_deployment(
|
||||
def _register_priced_deployment(
|
||||
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
|
||||
) -> str:
|
||||
from litellm import Router
|
||||
|
|
@ -1073,8 +1073,8 @@ def _register_priced_vertex_deployment(
|
|||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-vertex-deployment",
|
||||
"litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"},
|
||||
"model_name": "priced-deployment",
|
||||
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
|
||||
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
|
||||
}
|
||||
]
|
||||
|
|
@ -1082,7 +1082,7 @@ def _register_priced_vertex_deployment(
|
|||
return router.model_list[0]["model_info"]["id"]
|
||||
|
||||
|
||||
def _vertex_claude_chat_response(
|
||||
def _claude_chat_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
usage: Final = Usage(
|
||||
|
|
@ -1094,7 +1094,7 @@ def _vertex_claude_chat_response(
|
|||
return ModelResponse(model=model, usage=usage), usage
|
||||
|
||||
|
||||
def _vertex_claude_messages_response(
|
||||
def _claude_messages_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
return {
|
||||
|
|
@ -1111,7 +1111,7 @@ def _vertex_claude_messages_response(
|
|||
}, None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
|
||||
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
||||
local_model_cost_map: None,
|
||||
|
|
@ -1119,7 +1119,7 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
|||
web_search_requests: int,
|
||||
search_cost_per_query: float,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
|
||||
response_object, usage = build_response(web_search_requests)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
@ -1133,37 +1133,40 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
|||
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "served_model"),
|
||||
("deployment_model", "served_model", "custom_llm_provider"),
|
||||
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
|
||||
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
|
||||
[
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5"),
|
||||
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929"),
|
||||
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001"),
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
|
||||
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
|
||||
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
|
||||
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
|
||||
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("deployment_id", [None, "vertex-claude-token-priced"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price(
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int, str], tuple[object, Usage | None]],
|
||||
deployment_model: str,
|
||||
served_model: str,
|
||||
custom_llm_provider: str,
|
||||
deployment_id: str | None,
|
||||
) -> None:
|
||||
served_search_pricing: Final = (
|
||||
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
|
||||
)
|
||||
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
|
||||
registered_id: Final = _register_priced_vertex_deployment(deployment_model, None, deployment_id)
|
||||
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
|
||||
response_object, usage = build_response(2, served_model)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=registered_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
|
|
@ -1174,7 +1177,7 @@ def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_searc
|
|||
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
|
|
@ -1193,9 +1196,14 @@ def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_mode
|
|||
assert cost == 0.0
|
||||
|
||||
|
||||
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None:
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
|
||||
local_model_cost_map: None, deployment_id: str | None
|
||||
) -> None:
|
||||
search_cost_per_prompt: Final = 0.03
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)
|
||||
registered_id: Final = _register_priced_deployment(
|
||||
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
|
||||
)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
|
|
@ -1204,7 +1212,7 @@ def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model
|
|||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
model=registered_id,
|
||||
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue