mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(cost-map): rolling provider pricing, capability and retirement audit (#48188)
* fix(cost-map): registry audit 2026-10-24, add provider retirement dates and fix mistral-large-4 and azure_ai sonnet 5.5 prices Co-authored-by: Hannes Brantner <desktop.stilted-9t@icloud.com> Co-authored-by: Aasif-Simpplr <aasif.multani@simpplr.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mistral): derive mistral-large-4 cost expectations from the registry row instead of pinned prices Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): add fireworks spot routers and reprice together Qwen3.7-Max Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): absorb open registry PRs and update azure_ai mistral-small-2503 retirement date Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Aryan <aryan.m@kelpglobal.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(cost-map): credit absorbed registry PR authors Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: Aryan <aryan.m@kelpglobal.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): add azure gpt-6.1-terra rows and fireworks models/rune path Co-authored-by: ceng-p <christianpetersen98@gmail.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): add azure_ai gpt-6.1-terra global standard row Co-authored-by: Ahmet Selim FEDAKAR <122402435+selimfedakar@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): price mistral-large-4 cache reads at the published $0.14 Co-authored-by: zoveki <349028329+zoveki@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * revert(cost-map): leave mistral-large-4 rows to #48825 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(cost-map): refresh provider pricing and retirement metadata Absorb the verified GPT-5.6 Terra and Luna price fields from #48709. Add official Groq caching, xAI batch pricing, Moonshot metadata, and retirement dates without replacing existing model entries. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Hannes Brantner <desktop.stilted-9t@icloud.com> Co-authored-by: Aasif-Simpplr <aasif.multani@simpplr.com> Co-authored-by: Aryan <aryan.m@kelpglobal.com> Co-authored-by: kerry <kerry@berri.ai> Co-authored-by: ceng-p <christianpetersen98@gmail.com> Co-authored-by: Ahmet Selim FEDAKAR <122402435+selimfedakar@users.noreply.github.com> Co-authored-by: zoveki <349028329+zoveki@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
db5b940143
commit
3afe779125
4 changed files with 1973 additions and 436 deletions
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -1,4 +1,3 @@
|
|||
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -19,9 +18,7 @@ def test_web_search_cost_low():
|
|||
web_search_options=web_search_options, model_info=model_info
|
||||
)
|
||||
|
||||
assert (
|
||||
cost == model_info["search_context_cost_per_query"]["search_context_size_low"]
|
||||
)
|
||||
assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"]
|
||||
|
||||
|
||||
def test_web_search_cost_medium():
|
||||
|
|
@ -32,10 +29,7 @@ def test_web_search_cost_medium():
|
|||
web_search_options=web_search_options, model_info=model_info
|
||||
)
|
||||
|
||||
assert (
|
||||
cost
|
||||
== model_info["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
)
|
||||
assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
|
||||
|
||||
def test_web_search_cost_high():
|
||||
|
|
@ -46,9 +40,7 @@ def test_web_search_cost_high():
|
|||
web_search_options=web_search_options, model_info=model_info
|
||||
)
|
||||
|
||||
assert (
|
||||
cost == model_info["search_context_cost_per_query"]["search_context_size_high"]
|
||||
)
|
||||
assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"]
|
||||
|
||||
|
||||
def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
|
||||
|
|
@ -60,9 +52,7 @@ def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
|
|||
|
||||
|
||||
def test_get_web_search_options_accepts_non_list_tool_iterables():
|
||||
options = StandardBuiltInToolCostTracking.get_web_search_options(
|
||||
{"tools": ({"type": "web_search_preview"},)}
|
||||
)
|
||||
options = StandardBuiltInToolCostTracking.get_web_search_options({"tools": ({"type": "web_search_preview"},)})
|
||||
|
||||
assert options == {"type": "web_search_preview"}
|
||||
|
||||
|
|
@ -76,25 +66,15 @@ def test_get_file_search_tool_call_does_not_validate_tool_payload():
|
|||
# Test file search cost calculation
|
||||
def test_file_search_cost():
|
||||
file_search = FileSearchTool(type="file_search")
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(
|
||||
file_search=file_search
|
||||
)
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search)
|
||||
assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call
|
||||
|
||||
|
||||
# Test edge cases
|
||||
def test_none_inputs():
|
||||
# Test with None inputs
|
||||
assert (
|
||||
StandardBuiltInToolCostTracking.get_cost_for_web_search(
|
||||
web_search_options=None, model_info=None
|
||||
)
|
||||
== 0.0
|
||||
)
|
||||
assert (
|
||||
StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None)
|
||||
== 0.0
|
||||
)
|
||||
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0
|
||||
assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0
|
||||
|
||||
|
||||
# Test the main get_cost_for_built_in_tools method
|
||||
|
|
@ -119,9 +99,7 @@ def test_get_cost_for_built_in_tools_file_search():
|
|||
Test that the cost for a file search is 0.00 when no response object is provided
|
||||
"""
|
||||
model = "gpt-4"
|
||||
standard_built_in_tools_params = StandardBuiltInToolsParams(
|
||||
file_search=FileSearchTool(type="file_search")
|
||||
)
|
||||
standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search"))
|
||||
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=model,
|
||||
|
|
@ -144,9 +122,7 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict():
|
|||
usage = Usage(server_tool_use={"web_search_requests": 1})
|
||||
|
||||
assert isinstance(usage.server_tool_use, ServerToolUse)
|
||||
assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
|
||||
response_object=None, usage=usage
|
||||
)
|
||||
assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage)
|
||||
|
||||
|
||||
def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero():
|
||||
|
|
@ -257,9 +233,7 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par
|
|||
)
|
||||
|
||||
assert web_search_cost > 0, "Web search cost should be non-zero"
|
||||
assert (
|
||||
cost >= web_search_cost
|
||||
), f"completion_cost ({cost}) should include web search cost ({web_search_cost})"
|
||||
assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -286,18 +260,14 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map):
|
|||
web_search_requests = 2
|
||||
model_info = litellm.get_model_info(model)
|
||||
assert model_info["web_search_billing_unit"] == "per_query"
|
||||
per_query_cost = model_info["search_context_cost_per_query"][
|
||||
"search_context_size_medium"
|
||||
]
|
||||
per_query_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
expected_cost = per_query_cost * web_search_requests
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=11,
|
||||
completion_tokens=100,
|
||||
total_tokens=111,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=11, web_search_requests=web_search_requests
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=web_search_requests),
|
||||
)
|
||||
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
@ -309,8 +279,7 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map):
|
|||
)
|
||||
|
||||
assert cost == pytest.approx(expected_cost), (
|
||||
f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} "
|
||||
f"per_query search fee, got ${cost}"
|
||||
f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} per_query search fee, got ${cost}"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -353,17 +322,13 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map):
|
|||
model = "vertex_ai/gemini-2.5-flash"
|
||||
model_info = litellm.get_model_info(model)
|
||||
assert not model_info.get("web_search_billing_unit")
|
||||
expected_cost = model_info["search_context_cost_per_query"][
|
||||
"search_context_size_medium"
|
||||
]
|
||||
expected_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
|
||||
usage = Usage(
|
||||
prompt_tokens=11,
|
||||
completion_tokens=100,
|
||||
total_tokens=111,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=11, web_search_requests=2
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2),
|
||||
)
|
||||
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
@ -375,8 +340,7 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map):
|
|||
)
|
||||
|
||||
assert cost == pytest.approx(expected_cost), (
|
||||
f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), "
|
||||
f"got ${cost}"
|
||||
f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), got ${cost}"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -402,9 +366,7 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model(
|
|||
prompt_tokens=11,
|
||||
completion_tokens=100,
|
||||
total_tokens=111,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=11, web_search_requests=2
|
||||
),
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2),
|
||||
)
|
||||
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
|
|
@ -427,7 +389,6 @@ def _openai_responses_with_web_search_calls(model, num_calls):
|
|||
ResponseFunctionWebSearch,
|
||||
)
|
||||
|
||||
|
||||
output = [
|
||||
ResponseFunctionWebSearch(
|
||||
id=f"ws_{i}",
|
||||
|
|
@ -458,9 +419,7 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m
|
|||
from litellm.types.utils import Usage
|
||||
|
||||
model = "gpt-4o-search-preview"
|
||||
per_call = litellm.get_model_info(model)["search_context_cost_per_query"][
|
||||
"search_context_size_medium"
|
||||
]
|
||||
per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
|
||||
|
||||
for num_calls in (1, 3):
|
||||
|
|
@ -487,9 +446,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
|
|||
from litellm.types.utils import Usage
|
||||
|
||||
model = "gpt-4o-search-preview"
|
||||
per_call = litellm.get_model_info(model)["search_context_cost_per_query"][
|
||||
"search_context_size_medium"
|
||||
]
|
||||
per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"]
|
||||
|
||||
response = ResponsesAPIResponse.model_validate(
|
||||
{
|
||||
|
|
@ -498,10 +455,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
|
|||
"model": model,
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"output": [
|
||||
{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"}
|
||||
for i in range(3)
|
||||
],
|
||||
"output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)],
|
||||
}
|
||||
)
|
||||
assert all(isinstance(item, dict) for item in response.output)
|
||||
|
|
@ -514,9 +468,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
|
|||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(3 * per_call), (
|
||||
f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}"
|
||||
)
|
||||
assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}"
|
||||
|
||||
|
||||
# Note: File search integration test removed due to complex annotation detection logic
|
||||
|
|
@ -596,8 +548,6 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
|
|||
_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
|
||||
|
||||
|
||||
|
||||
|
||||
def _openai_responses_response(model, output, usage=None, tools=None):
|
||||
return ResponsesAPIResponse.model_validate(
|
||||
{
|
||||
|
|
@ -815,6 +765,31 @@ def test_responses_usage_tool_usage_web_search_billed_without_output_item(local_
|
|||
assert cost == pytest.approx(num_requests * per_call)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["azure/gpt-5-mini", "azure/gpt-5.6-luna"])
|
||||
def test_azure_responses_web_search_billed_per_bing_request(local_model_cost_map, model):
|
||||
"""Azure OpenAI web_search is Grounding with Bing: $0.014 per reported Bing request, including rows
|
||||
whose OpenAI twin prices search but the Azure row used to bill it at $0."""
|
||||
for num_requests in (1, 5):
|
||||
response = _openai_responses_response(
|
||||
model,
|
||||
[dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)],
|
||||
usage={
|
||||
"input_tokens": 10,
|
||||
"output_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
"tool_usage": {"web_search": {"num_requests": num_requests}},
|
||||
},
|
||||
)
|
||||
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=model,
|
||||
response_object=response,
|
||||
usage=None,
|
||||
custom_llm_provider="azure",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
assert cost == pytest.approx(num_requests * 0.014)
|
||||
|
||||
|
||||
def _image_gen_token_usage(input_text, input_image, output_image, output_text):
|
||||
return {
|
||||
"input_tokens": 10,
|
||||
|
|
@ -1012,6 +987,8 @@ def test_responses_image_gen_total_without_token_details_falls_back_to_per_image
|
|||
)
|
||||
)
|
||||
assert cost > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_key", "web_search_requests", "medium_cost_per_query", "expected_cost"),
|
||||
[
|
||||
|
|
|
|||
|
|
@ -224,6 +224,41 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict):
|
|||
)
|
||||
|
||||
|
||||
# Azure OpenAI web_search runs on Grounding with Bing and bills each Bing request reported in
|
||||
# tool_usage.web_search.num_requests: https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/web-search
|
||||
# Grounding with Bing lists $14 per 1,000 transactions: https://www.microsoft.com/en-us/bing/apis/grounding-pricing
|
||||
AZURE_WEB_SEARCH_COST_PER_REQUEST: Final = 0.014
|
||||
_AZURE_DEPLOYMENT_PREFIX: Final = re.compile(r"^azure/(us/|eu/|global/|global-standard/)?")
|
||||
|
||||
|
||||
def _azure_rows(prices: Mapping[str, object]) -> Mapping[str, dict]:
|
||||
return MappingProxyType(
|
||||
{key: row for key, row in prices.items() if key.startswith("azure/") and isinstance(row, dict)}
|
||||
)
|
||||
|
||||
|
||||
def test_azure_web_search_rows_charge_the_grounding_with_bing_price(prices: dict):
|
||||
wrong = {
|
||||
key: row["search_context_cost_per_query"]
|
||||
for key, row in _azure_rows(prices).items()
|
||||
if row.get("search_context_cost_per_query")
|
||||
and set(row["search_context_cost_per_query"].values()) != {AZURE_WEB_SEARCH_COST_PER_REQUEST}
|
||||
}
|
||||
assert wrong == {}
|
||||
|
||||
|
||||
def test_azure_rows_price_web_search_when_their_openai_model_does(prices: dict):
|
||||
missing = sorted(
|
||||
key
|
||||
for key, row in _azure_rows(prices).items()
|
||||
if not row.get("search_context_cost_per_query")
|
||||
and row.get("mode") in ("chat", "responses", None)
|
||||
and isinstance(twin := prices.get(_AZURE_DEPLOYMENT_PREFIX.sub("", key)), dict)
|
||||
and twin.get("search_context_cost_per_query")
|
||||
)
|
||||
assert missing == []
|
||||
|
||||
|
||||
PRIORITY_SUFFIX: Final = "_priority"
|
||||
MIN_PRIORITY_MARKUP: Final = 1.0
|
||||
MAX_PRIORITY_MARKUP: Final = 5.0
|
||||
|
|
@ -845,7 +880,10 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa
|
|||
assert entry["cache_read_input_token_cost"] == pytest.approx(1.2 / 6.9565 / 1_000_000, rel=1e-4)
|
||||
sibling = rows["volcengine/doubao-seed-2-1-pro-260628"]
|
||||
assert isinstance(sibling, dict)
|
||||
assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"))
|
||||
assert all(
|
||||
entry[key] == sibling[key]
|
||||
for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue