mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(vertex_ai): bill token-priced Claude deployments at the served model's search price
A Vertex Claude deployment priced in model_info without a search price now bills each web search at the served model's cost map price, read at cost time and never written into the deployment entry. An explicit search price, zero included, still wins.
This commit is contained in:
parent
112ed5c534
commit
9f894486c9
4 changed files with 101 additions and 14 deletions
|
|
@ -215,6 +215,7 @@ class StandardBuiltInToolCostTracking:
|
|||
usage=resolved_usage,
|
||||
model_info=model_info,
|
||||
response_model=response_model if isinstance(response_model, str) else None,
|
||||
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
|
|||
from . import *
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
)
|
||||
|
|
@ -30,8 +32,25 @@ def get_cost_for_google_maps_grounding_request(
|
|||
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
|
||||
|
||||
|
||||
def _with_served_model_search_pricing(
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
|
||||
) -> "ModelInfo":
|
||||
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
|
||||
return model_info
|
||||
served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None
|
||||
if served_model_info is None:
|
||||
return model_info
|
||||
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(
|
||||
custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo", response_model: str | None = None
|
||||
custom_llm_provider: str,
|
||||
usage: "Usage",
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None = None,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
|
||||
) -> float | None:
|
||||
"""
|
||||
Get the cost for a web search request for a given model.
|
||||
|
|
@ -41,6 +60,7 @@ def get_cost_for_web_search_request(
|
|||
usage: The usage object.
|
||||
model_info: The model info.
|
||||
response_model: The model the provider response names.
|
||||
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
|
||||
"""
|
||||
if custom_llm_provider == "gemini":
|
||||
from .gemini.cost_calculator import cost_per_web_search_request
|
||||
|
|
@ -59,7 +79,10 @@ def get_cost_for_web_search_request(
|
|||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
from .vertex_ai.gemini.cost_calculator import (
|
||||
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
|
||||
|
|
|
|||
|
|
@ -3388,8 +3388,6 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
|||
)
|
||||
|
||||
|
||||
|
||||
# vertex_ai/claude-sonnet-5-5 deployment priced in its model_info, so it bills under the deployment's own id
|
||||
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
|
||||
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
|
||||
)
|
||||
|
|
@ -3517,6 +3515,20 @@ VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
|
|||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.034,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
|
||||
"/v1/chat/completions",
|
||||
|
|
@ -3943,6 +3955,7 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
|
||||
|
|
|
|||
|
|
@ -1053,23 +1053,27 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
|
|||
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
|
||||
|
||||
|
||||
def _register_priced_vertex_deployment(model: str, search_cost_per_query: float) -> str:
|
||||
def _register_priced_vertex_deployment(model: str, search_cost_per_query: float | None) -> str:
|
||||
from litellm import Router
|
||||
|
||||
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
|
||||
search_rates: Final = (
|
||||
{}
|
||||
if search_cost_per_query is None
|
||||
else {
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": search_cost_per_query,
|
||||
"search_context_size_medium": search_cost_per_query,
|
||||
"search_context_size_high": search_cost_per_query,
|
||||
}
|
||||
}
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-vertex-deployment",
|
||||
"litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"},
|
||||
"model_info": {
|
||||
"input_cost_per_token": 3e-6,
|
||||
"output_cost_per_token": 1.5e-5,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": search_cost_per_query,
|
||||
"search_context_size_medium": search_cost_per_query,
|
||||
"search_context_size_high": search_cost_per_query,
|
||||
},
|
||||
},
|
||||
"model_info": {**token_rates, **search_rates},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
|
@ -1123,6 +1127,52 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
|||
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response])
|
||||
def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int], tuple[object, Usage | None]],
|
||||
) -> None:
|
||||
served_search_pricing: Final = (
|
||||
litellm.get_model_info(model="vertex_ai/claude-sonnet-5-5").get("search_context_cost_per_query") or {}
|
||||
)
|
||||
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
response_object, usage = build_response(2)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert served_search_price > 0
|
||||
assert cost == pytest.approx(2 * served_search_price)
|
||||
|
||||
|
||||
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=2),
|
||||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
|
||||
|
||||
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None:
|
||||
search_cost_per_prompt: Final = 0.03
|
||||
deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue