From 1ca8c8c94a745f8334b5fc7df0908af91cda9388 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:45:31 -0700 Subject: [PATCH] fix(cost): bill served-model search price on token-only Anthropic and Azure AI Claude deployments Token-only Claude deployments on anthropic and azure_ai billed web search at 0 because the deployment entry has no search price. They now fall back to the served model's cost map search price, as vertex_ai already does. Vertex routing reads the served model first, so a Gemini deployment whose id contains claude keeps per-prompt grounding --- litellm/llms/__init__.py | 21 ++++++-- .../test_tool_call_cost_tracking.py | 48 +++++++++++-------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index 95a9e689290..84c91be9386 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -35,11 +35,14 @@ def get_cost_for_google_maps_grounding_request( def _with_served_model_search_pricing( model_info: "ModelInfo", response_model: str | None, + served_model_provider: str, lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", ) -> "ModelInfo": if model_info.get("search_context_cost_per_query") is not None or response_model is None: return model_info - served_model_info: Final = lookup_model_info(response_model, "vertex_ai") if lookup_model_info is not None else None + served_model_info: Final = ( + lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None + ) if served_model_info is None: return model_info return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} @@ -69,18 +72,23 @@ def get_cost_for_web_search_request( elif custom_llm_provider == "anthropic": from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info), + usage=usage, + ) elif custom_llm_provider.startswith("vertex_ai"): # Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests # (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests # (which is the Gemini field). Route claude-* models to the Anthropic calculator. model_key: Final[str] = model_info.get("key", "") if model_info else "" - if "claude" in model_key.lower() or "claude" in (response_model or "").lower(): + if "claude" in (response_model or model_key).lower(): from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") return get_cost_for_anthropic_web_search( - model_info=_with_served_model_search_pricing(model_info, response_model, lookup_model_info), + model_info=_with_served_model_search_pricing( + model_info, response_model, "vertex_ai", lookup_model_info + ), usage=usage, ) @@ -97,7 +105,10 @@ def get_cost_for_web_search_request( return None from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info), + usage=usage, + ) elif custom_llm_provider == "perplexity": # Perplexity handles search costs internally in its own cost calculator # Return 0.0 to indicate costs are already accounted for diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 71dae8ee867..9da2632943c 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1053,7 +1053,7 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None -def _register_priced_vertex_deployment( +def _register_priced_deployment( model: str, search_cost_per_query: float | None, deployment_id: str | None = None ) -> str: from litellm import Router @@ -1073,8 +1073,8 @@ def _register_priced_vertex_deployment( router: Final = Router( model_list=[ { - "model_name": "priced-vertex-deployment", - "litellm_params": {"model": model, "vertex_project": "p", "vertex_location": "global"}, + "model_name": "priced-deployment", + "litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"}, "model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})}, } ] @@ -1082,7 +1082,7 @@ def _register_priced_vertex_deployment( return router.model_list[0]["model_info"]["id"] -def _vertex_claude_chat_response( +def _claude_chat_response( web_search_requests: int, model: str = "claude-sonnet-5-5" ) -> tuple[object, Usage | None]: usage: Final = Usage( @@ -1094,7 +1094,7 @@ def _vertex_claude_chat_response( return ModelResponse(model=model, usage=usage), usage -def _vertex_claude_messages_response( +def _claude_messages_response( web_search_requests: int, model: str = "claude-sonnet-5-5" ) -> tuple[object, Usage | None]: return { @@ -1111,7 +1111,7 @@ def _vertex_claude_messages_response( }, None -@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response]) +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) @pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)]) def test_vertex_claude_deployment_priced_web_search_billed_per_request( local_model_cost_map: None, @@ -1119,7 +1119,7 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request( web_search_requests: int, search_cost_per_query: float, ) -> None: - deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query) + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query) response_object, usage = build_response(web_search_requests) cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -1133,37 +1133,40 @@ def test_vertex_claude_deployment_priced_web_search_billed_per_request( assert cost == pytest.approx(web_search_requests * search_cost_per_query) -@pytest.mark.parametrize("build_response", [_vertex_claude_chat_response, _vertex_claude_messages_response]) +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) @pytest.mark.parametrize( - ("deployment_model", "served_model"), + ("deployment_model", "served_model", "custom_llm_provider"), # Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too: # https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30) [ - ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5"), - ("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929"), - ("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001"), + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"), + ("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), ], ) -@pytest.mark.parametrize("deployment_id", [None, "vertex-claude-token-priced"], ids=["hashed_id", "claude_in_id"]) -def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_search_price( +@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"]) +def test_claude_deployment_with_token_rates_only_bills_served_model_search_price( local_model_cost_map: None, build_response: Callable[[int, str], tuple[object, Usage | None]], deployment_model: str, served_model: str, + custom_llm_provider: str, deployment_id: str | None, ) -> None: served_search_pricing: Final = ( litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {} ) served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 - registered_id: Final = _register_priced_vertex_deployment(deployment_model, None, deployment_id) + registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id) response_object, usage = build_response(2, served_model) cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=registered_id, response_object=response_object, usage=usage, - custom_llm_provider="vertex_ai", + custom_llm_provider=custom_llm_provider, standard_built_in_tools_params=None, ) @@ -1174,7 +1177,7 @@ def test_vertex_claude_deployment_with_token_rates_only_bills_served_model_searc def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( local_model_cost_map: None, ) -> None: - deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/claude-sonnet-5-5", None) + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None) usage: Final = Usage( prompt_tokens=1840, completion_tokens=412, @@ -1193,9 +1196,14 @@ def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_mode assert cost == 0.0 -def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model_cost_map: None) -> None: +@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"]) +def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt( + local_model_cost_map: None, deployment_id: str | None +) -> None: search_cost_per_prompt: Final = 0.03 - deployment_id: Final = _register_priced_vertex_deployment("vertex_ai/gemini-2.5-flash", search_cost_per_prompt) + registered_id: Final = _register_priced_deployment( + "vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id + ) usage: Final = Usage( prompt_tokens=1840, completion_tokens=412, @@ -1204,7 +1212,7 @@ def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(local_model ) cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( - model=deployment_id, + model=registered_id, response_object=ModelResponse(model="gemini-2.5-flash", usage=usage), usage=usage, custom_llm_provider="vertex_ai",