This commit is contained in:
devin-ai-integration[bot] 2026-10-27 19:12:48 +00:00 • committed by GitHub
commit e922ebfeb6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 1879 additions and 429 deletions

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,3 @@
from typing import Final
import pytest
@ -19,9 +18,7 @@ def test_web_search_cost_low():
web_search_options=web_search_options, model_info=model_info
)
assert (
cost == model_info["search_context_cost_per_query"]["search_context_size_low"]
)
assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"]
def test_web_search_cost_medium():
@ -32,10 +29,7 @@ def test_web_search_cost_medium():
web_search_options=web_search_options, model_info=model_info
)
assert (
cost
== model_info["search_context_cost_per_query"]["search_context_size_medium"]
)
assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"]
def test_web_search_cost_high():
@ -46,9 +40,7 @@ def test_web_search_cost_high():
web_search_options=web_search_options, model_info=model_info
)
assert (
cost == model_info["search_context_cost_per_query"]["search_context_size_high"]
)
assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"]
def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
@ -60,9 +52,7 @@ def test_get_web_search_options_reads_tools_when_options_are_null() -> None:
def test_get_web_search_options_accepts_non_list_tool_iterables():
options = StandardBuiltInToolCostTracking.get_web_search_options(
{"tools": ({"type": "web_search_preview"},)}
)
options = StandardBuiltInToolCostTracking.get_web_search_options({"tools": ({"type": "web_search_preview"},)})
assert options == {"type": "web_search_preview"}
@ -76,25 +66,15 @@ def test_get_file_search_tool_call_does_not_validate_tool_payload():
# Test file search cost calculation
def test_file_search_cost():
file_search = FileSearchTool(type="file_search")
cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(
file_search=file_search
)
cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search)
assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call
# Test edge cases
def test_none_inputs():
# Test with None inputs
assert (
StandardBuiltInToolCostTracking.get_cost_for_web_search(
web_search_options=None, model_info=None
)
== 0.0
)
assert (
StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None)
== 0.0
)
assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0
assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0
# Test the main get_cost_for_built_in_tools method
@ -119,9 +99,7 @@ def test_get_cost_for_built_in_tools_file_search():
Test that the cost for a file search is 0.00 when no response object is provided
"""
model = "gpt-4"
standard_built_in_tools_params = StandardBuiltInToolsParams(
file_search=FileSearchTool(type="file_search")
)
standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search"))
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
@ -144,9 +122,7 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict():
usage = Usage(server_tool_use={"web_search_requests": 1})
assert isinstance(usage.server_tool_use, ServerToolUse)
assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=None, usage=usage
)
assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage)
def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero():
@ -257,9 +233,7 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par
)
assert web_search_cost > 0, "Web search cost should be non-zero"
assert (
cost >= web_search_cost
), f"completion_cost ({cost}) should include web search cost ({web_search_cost})"
assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})"
@pytest.mark.parametrize(
@ -286,18 +260,14 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map):
web_search_requests = 2
model_info = litellm.get_model_info(model)
assert model_info["web_search_billing_unit"] == "per_query"
per_query_cost = model_info["search_context_cost_per_query"][
"search_context_size_medium"
]
per_query_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"]
expected_cost = per_query_cost * web_search_requests
usage = Usage(
prompt_tokens=11,
completion_tokens=100,
total_tokens=111,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=11, web_search_requests=web_search_requests
),
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=web_search_requests),
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
@ -309,8 +279,7 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map):
)
assert cost == pytest.approx(expected_cost), (
f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} "
f"per_query search fee, got ${cost}"
f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} per_query search fee, got ${cost}"
)
@ -353,17 +322,13 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map):
model = "vertex_ai/gemini-2.5-flash"
model_info = litellm.get_model_info(model)
assert not model_info.get("web_search_billing_unit")
expected_cost = model_info["search_context_cost_per_query"][
"search_context_size_medium"
]
expected_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"]
usage = Usage(
prompt_tokens=11,
completion_tokens=100,
total_tokens=111,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=11, web_search_requests=2
),
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2),
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
@ -375,8 +340,7 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map):
)
assert cost == pytest.approx(expected_cost), (
f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), "
f"got ${cost}"
f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), got ${cost}"
)
@ -402,9 +366,7 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model(
prompt_tokens=11,
completion_tokens=100,
total_tokens=111,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=11, web_search_requests=2
),
prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2),
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
@ -427,7 +389,6 @@ def _openai_responses_with_web_search_calls(model, num_calls):
ResponseFunctionWebSearch,
)
output = [
ResponseFunctionWebSearch(
id=f"ws_{i}",
@ -458,9 +419,7 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m
from litellm.types.utils import Usage
model = "gpt-4o-search-preview"
per_call = litellm.get_model_info(model)["search_context_cost_per_query"][
"search_context_size_medium"
]
per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"]
usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15)
for num_calls in (1, 3):
@ -487,9 +446,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
from litellm.types.utils import Usage
model = "gpt-4o-search-preview"
per_call = litellm.get_model_info(model)["search_context_cost_per_query"][
"search_context_size_medium"
]
per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"]
response = ResponsesAPIResponse.model_validate(
{
@ -498,10 +455,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
"model": model,
"object": "response",
"status": "completed",
"output": [
{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"}
for i in range(3)
],
"output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)],
}
)
assert all(isinstance(item, dict) for item in response.output)
@ -514,9 +468,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map):
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(3 * per_call), (
f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}"
)
assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}"
# Note: File search integration test removed due to complex annotation detection logic
@ -596,8 +548,6 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
def _openai_responses_response(model, output, usage=None, tools=None):
return ResponsesAPIResponse.model_validate(
{
@ -815,6 +765,31 @@ def test_responses_usage_tool_usage_web_search_billed_without_output_item(local_
assert cost == pytest.approx(num_requests * per_call)
@pytest.mark.parametrize("model", ["azure/gpt-5-mini", "azure/gpt-5.6-luna"])
def test_azure_responses_web_search_billed_per_bing_request(local_model_cost_map, model):
"""Azure OpenAI web_search is Grounding with Bing: $0.014 per reported Bing request, including rows
whose OpenAI twin prices search but the Azure row used to bill it at $0."""
for num_requests in (1, 5):
response = _openai_responses_response(
model,
[dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)],
usage={
"input_tokens": 10,
"output_tokens": 5,
"total_tokens": 15,
"tool_usage": {"web_search": {"num_requests": num_requests}},
},
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=response,
usage=None,
custom_llm_provider="azure",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(num_requests * 0.014)
def _image_gen_token_usage(input_text, input_image, output_image, output_text):
return {
"input_tokens": 10,
@ -1012,6 +987,8 @@ def test_responses_image_gen_total_without_token_details_falls_back_to_per_image
)
)
assert cost > 0
@pytest.mark.parametrize(
("model_key", "web_search_requests", "medium_cost_per_query", "expected_cost"),
[

View file

@ -32,9 +32,8 @@ def test_metadata_and_backup(model):
assert info["litellm_provider"] == "mistral"
assert info["mode"] == "chat"
assert info["max_input_tokens"] == 1048576
assert info["input_cost_per_token"] == 6.8e-07
assert info["output_cost_per_token"] == 2.09e-06
assert info["cache_read_input_token_cost"] == 6.8e-08
assert info["output_cost_per_token"] > info["input_cost_per_token"] > 0
assert info["cache_read_input_token_cost"] == pytest.approx(info["input_cost_per_token"] / 10)
assert info["reasoning_effort_levels"] == ["none", "high"]
for capability in (
"supports_vision",
@ -59,12 +58,16 @@ def test_reasoning_effort(model, effort):
@pytest.mark.parametrize("model", MODELS)
def test_completion_cost(model):
info = litellm.get_model_info(model)
input_rate = info["input_cost_per_token"]
output_rate = info["output_cost_per_token"]
cache_read_rate = info["cache_read_input_token_cost"]
uncached_response = litellm.ModelResponse(
model=model.split("/", 1)[1],
usage={"prompt_tokens": 1000, "completion_tokens": 100, "total_tokens": 1100},
)
assert litellm.completion_cost(completion_response=uncached_response, model=model) == pytest.approx(
1000 * 6.8e-07 + 100 * 2.09e-06
1000 * input_rate + 100 * output_rate
)
response = litellm.ModelResponse(
model=model.split("/", 1)[1],
@ -76,7 +79,7 @@ def test_completion_cost(model):
},
)
assert litellm.completion_cost(completion_response=response, model=model) == pytest.approx(
400 * 6.8e-07 + 600 * 6.8e-08 + 100 * 2.09e-06
400 * input_rate + 600 * cache_read_rate + 100 * output_rate
)

View file

@ -224,6 +224,41 @@ def test_dated_variants_carry_base_alias_service_tier_pricing(prices: dict):
)
# Azure OpenAI web_search runs on Grounding with Bing and bills each Bing request reported in
# tool_usage.web_search.num_requests: https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/web-search
# Grounding with Bing lists $14 per 1,000 transactions: https://www.microsoft.com/en-us/bing/apis/grounding-pricing
AZURE_WEB_SEARCH_COST_PER_REQUEST: Final = 0.014
_AZURE_DEPLOYMENT_PREFIX: Final = re.compile(r"^azure/(us/|eu/|global/|global-standard/)?")
def _azure_rows(prices: Mapping[str, object]) -> Mapping[str, dict]:
return MappingProxyType(
{key: row for key, row in prices.items() if key.startswith("azure/") and isinstance(row, dict)}
)
def test_azure_web_search_rows_charge_the_grounding_with_bing_price(prices: dict):
wrong = {
key: row["search_context_cost_per_query"]
for key, row in _azure_rows(prices).items()
if row.get("search_context_cost_per_query")
and set(row["search_context_cost_per_query"].values()) != {AZURE_WEB_SEARCH_COST_PER_REQUEST}
}
assert wrong == {}
def test_azure_rows_price_web_search_when_their_openai_model_does(prices: dict):
missing = sorted(
key
for key, row in _azure_rows(prices).items()
if not row.get("search_context_cost_per_query")
and row.get("mode") in ("chat", "responses", None)
and isinstance(twin := prices.get(_AZURE_DEPLOYMENT_PREFIX.sub("", key)), dict)
and twin.get("search_context_cost_per_query")
)
assert missing == []
# Azure Foundry GPT-6.1 Sol announcement, 2026-09-29: https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-gpt-6-1-sol-in-microsoft-foundry-advanced-intelligence-optimized-for/4560811
AZURE_GPT_6_1_SOL_REGIONAL_PREMIUM: Final = MappingProxyType({"us": 1.1, "eu": 1.2, "apac": 1.2})
@ -801,7 +836,10 @@ def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(pa
assert entry["cache_read_input_token_cost"] == pytest.approx(1.2 / 6.9565 / 1_000_000, rel=1e-4)
sibling = rows["volcengine/doubao-seed-2-1-pro-260628"]
assert isinstance(sibling, dict)
assert all(entry[key] == sibling[key] for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost"))
assert all(
entry[key] == sibling[key]
for key in ("input_cost_per_token", "output_cost_per_token", "cache_read_input_token_cost")
)
def _bedrock_gpt_oss_rows(prices: Mapping[str, JsonValue]) -> tuple[str, ...]: