From 24026ae0fe840d74b5c045353e33115386ef7fb0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:29:45 +0000 Subject: [PATCH 01/10] fix(cost): bill Responses API image_generation_call tool usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 102 +++++++++-- .../test_tool_call_cost_tracking.py | 173 ++++++++++++++++++ 2 files changed, 259 insertions(+), 16 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index bf99035a6b1..c241577c538 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -2,12 +2,13 @@ Helper utilities for tracking the cost of built-in tools. """ -from collections.abc import Mapping -from typing import Final, Literal +from collections.abc import Callable, Mapping +from typing import Final, Literal, cast from pydantic import ValidationError import litellm +from litellm._logging import verbose_logger from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS from litellm.litellm_core_utils.llm_cost_calc.utils import ( get_web_search_requests_from_usage, @@ -30,8 +31,15 @@ from litellm.types.utils import ( ) +def _output_item_field(output_item: object, field: str) -> object: + if isinstance(output_item, dict): + fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) + return fields.get(field) + return getattr(output_item, field, None) + + def _output_item_type(output_item: object) -> str | None: - item_type: Final = output_item.get("type") if isinstance(output_item, dict) else getattr(output_item, "type", None) + item_type: Final = _output_item_field(output_item, "type") return item_type if isinstance(item_type, str) else None @@ -87,31 +95,48 @@ class StandardBuiltInToolCostTracking: usage=usage, ) + image_generation_cost: Final = StandardBuiltInToolCostTracking._handle_image_generation_cost( + response_object=response_object, + custom_llm_provider=custom_llm_provider, + ) + # Handle web search if StandardBuiltInToolCostTracking.response_object_includes_web_search_call( response_object=response_object, usage=usage ): - return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost( - model=model, - custom_llm_provider=custom_llm_provider, - usage=usage, - standard_built_in_tools_params=standard_built_in_tools_params, - response_object=response_object, + return ( + google_maps_grounding_cost + + image_generation_cost + + StandardBuiltInToolCostTracking._handle_web_search_cost( + model=model, + custom_llm_provider=custom_llm_provider, + usage=usage, + standard_built_in_tools_params=standard_built_in_tools_params, + response_object=response_object, + ) ) # Handle file search if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object): - return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost( - model=model, - custom_llm_provider=custom_llm_provider, - standard_built_in_tools_params=standard_built_in_tools_params, + return ( + google_maps_grounding_cost + + image_generation_cost + + StandardBuiltInToolCostTracking._handle_file_search_cost( + model=model, + custom_llm_provider=custom_llm_provider, + standard_built_in_tools_params=standard_built_in_tools_params, + ) ) # Handle Azure assistant features - return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs( + return ( + google_maps_grounding_cost + + image_generation_cost + + StandardBuiltInToolCostTracking._handle_azure_assistant_costs( model=model, custom_llm_provider=custom_llm_provider, - standard_built_in_tools_params=standard_built_in_tools_params, + standard_built_in_tools_params=standard_built_in_tools_params, + ) ) @staticmethod @@ -210,6 +235,51 @@ class StandardBuiltInToolCostTracking: ) return max(count, 1) + @staticmethod + def response_object_includes_image_generation_call(response_object: object) -> bool: + """Check if the response object includes an image generation call (Responses API).""" + if not isinstance(response_object, ResponsesAPIResponse): + return False + return StandardBuiltInToolCostTracking.response_includes_output_type( + response_object=response_object, output_type="image_generation_call" + ) + + @staticmethod + def _image_generation_call_cost(output_item: object, custom_llm_provider: str | None) -> float: + from litellm.cost_calculator import ( + default_image_cost_calculator, # pyright: ignore[reportUnknownVariableType] # optional_params param is untyped + ) + + status: Final = _output_item_field(output_item, "status") + if status != "completed": + return 0.0 + quality: Final = _output_item_field(output_item, "quality") + size: Final = _output_item_field(output_item, "size") + try: + # the Responses image_generation tool item does not report the model, so price with + # gpt-image-1, OpenAI's default model for that tool + return cast(Callable[..., float], default_image_cost_calculator)( + model="gpt-image-1", + custom_llm_provider=custom_llm_provider or "openai", + quality=quality if isinstance(quality, str) and quality != "auto" else None, + n=1, + size=size if isinstance(size, str) else None, + ) + except Exception as e: + verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e) + return 0.0 + + @staticmethod + def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float: + if not isinstance(response_object, ResponsesAPIResponse): + return 0.0 + output: Final[list[object]] = cast(list[object], response_object.output) + return sum( + StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, custom_llm_provider) + for output_item in output + if _output_item_type(output_item) == "image_generation_call" + ) + @staticmethod def _handle_file_search_cost( model: str, @@ -505,7 +575,7 @@ class StandardBuiltInToolCostTracking: @staticmethod def response_includes_output_type( response_object: ResponsesAPIResponse, - output_type: Literal["web_search_call", "file_search_call"], + output_type: Literal["web_search_call", "file_search_call", "image_generation_call"], ) -> bool: """ Check if the ResponsesAPIResponse includes one of the specified output types. diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 7bae2eaa338..3ca7f153033 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -695,3 +695,176 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 + + +def _openai_responses_response(model, output): + return ResponsesAPIResponse.model_validate( + { + "id": "resp_1", + "created_at": 1754900000, + "model": model, + "object": "response", + "status": "completed", + "output": output, + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + ) + + +_ASSISTANT_MESSAGE_OUTPUT_ITEM = { + "type": "message", + "id": "msg_1", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "done", "annotations": []}], +} + +_GPT_IMAGE_1_HIGH_1024_COST_KEY = "high/1024-x-1024/gpt-image-1" + + +def test_responses_image_generation_call_billed_as_tool_usage_cost(local_model_cost_map): + """ + Regression: a Responses API output carrying an image_generation_call item was + charged $0 of tool usage because get_cost_for_built_in_tools only looked for + web/file search calls. A completed image_generation_call must be billed at the + gpt-image-1 rate for its reported quality and size. + """ + expected_image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] + response = _openai_responses_response( + "gpt-5", + [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": "high", + "size": "1024x1024", + "result": "AAAA", + }, + dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM), + ], + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + assert cost > 0 + assert cost == pytest.approx(expected_image_cost) + + +def test_responses_web_search_and_image_generation_costs_are_additive(local_model_cost_map): + """ + Regression: get_cost_for_built_in_tools returned early after the web search + branch, so a response billed for web search never reached image generation + pricing. A response with both must bill both. + """ + model = "gpt-4o-search-preview" + image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] + image_item = { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": "high", + "size": "1024x1024", + "result": "AAAA", + } + web_search_item = {"type": "web_search_call", "id": "ws_1", "status": "completed"} + + combined = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=_openai_responses_response(model, [web_search_item, image_item]), + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + web_search_only = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=_openai_responses_response(model, [web_search_item]), + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + image_only = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=_openai_responses_response(model, [image_item]), + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + assert image_only == pytest.approx(image_cost) + assert web_search_only > 0 + assert combined == pytest.approx(web_search_only + image_only) + + +def test_responses_incomplete_image_generation_call_not_billed(local_model_cost_map): + """A failed image_generation_call produced no billable image, so it must cost $0.""" + response = _openai_responses_response( + "gpt-5", + [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "failed", + "quality": "high", + "size": "1024x1024", + "result": None, + }, + dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM), + ], + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + +def test_completion_cost_includes_responses_image_generation_tool_cost(local_model_cost_map): + """ + An image_generation_call in the Responses output must flow through + completion_cost: the billed total for the same response without the image + item is the token-only baseline the image item must exceed by its tool cost. + """ + image_item = { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": "high", + "size": "1024x1024", + "result": "AAAA", + } + response_with_image = _openai_responses_response( + "gpt-5", [image_item, dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)] + ) + response_without_image = _openai_responses_response( + "gpt-5", [dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)] + ) + + cost_with_image = litellm.completion_cost( + completion_response=response_with_image, + model="gpt-5", + custom_llm_provider="openai", + call_type="aresponses", + ) + cost_without_image = litellm.completion_cost( + completion_response=response_without_image, + model="gpt-5", + custom_llm_provider="openai", + call_type="aresponses", + ) + + assert cost_with_image > cost_without_image + assert cost_with_image - cost_without_image == pytest.approx( + litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] + ) From 771355a7073adf9bf6e4789e9ad37d6b566d24ad Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:31:18 +0000 Subject: [PATCH 02/10] style(cost): tidy image generation tool cost helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 10 +- .../test_tool_call_cost_tracking.py | 112 ++++-------------- 2 files changed, 30 insertions(+), 92 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index c241577c538..1d4576a1511 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -2,7 +2,7 @@ Helper utilities for tracking the cost of built-in tools. """ -from collections.abc import Callable, Mapping +from collections.abc import Mapping from typing import Final, Literal, cast from pydantic import ValidationError @@ -133,8 +133,8 @@ class StandardBuiltInToolCostTracking: google_maps_grounding_cost + image_generation_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs( - model=model, - custom_llm_provider=custom_llm_provider, + model=model, + custom_llm_provider=custom_llm_provider, standard_built_in_tools_params=standard_built_in_tools_params, ) ) @@ -256,9 +256,7 @@ class StandardBuiltInToolCostTracking: quality: Final = _output_item_field(output_item, "quality") size: Final = _output_item_field(output_item, "size") try: - # the Responses image_generation tool item does not report the model, so price with - # gpt-image-1, OpenAI's default model for that tool - return cast(Callable[..., float], default_image_cost_calculator)( + return default_image_cost_calculator( model="gpt-image-1", custom_llm_provider=custom_llm_provider or "openai", quality=quality if isinstance(quality, str) and quality != "auto" else None, diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 3ca7f153033..c3a5698bf69 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,4 +1,3 @@ - import pytest import litellm @@ -17,9 +16,7 @@ def test_web_search_cost_low(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost == model_info["search_context_cost_per_query"]["search_context_size_low"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"] def test_web_search_cost_medium(): @@ -30,10 +27,7 @@ def test_web_search_cost_medium(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost - == model_info["search_context_cost_per_query"]["search_context_size_medium"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"] def test_web_search_cost_high(): @@ -44,33 +38,21 @@ def test_web_search_cost_high(): web_search_options=web_search_options, model_info=model_info ) - assert ( - cost == model_info["search_context_cost_per_query"]["search_context_size_high"] - ) + assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"] # Test file search cost calculation def test_file_search_cost(): file_search = FileSearchTool(type="file_search") - cost = StandardBuiltInToolCostTracking.get_cost_for_file_search( - file_search=file_search - ) + cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search) assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call # Test edge cases def test_none_inputs(): # Test with None inputs - assert ( - StandardBuiltInToolCostTracking.get_cost_for_web_search( - web_search_options=None, model_info=None - ) - == 0.0 - ) - assert ( - StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) - == 0.0 - ) + assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0 + assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0 # Test the main get_cost_for_built_in_tools method @@ -95,9 +77,7 @@ def test_get_cost_for_built_in_tools_file_search(): Test that the cost for a file search is 0.00 when no response object is provided """ model = "gpt-4" - standard_built_in_tools_params = StandardBuiltInToolsParams( - file_search=FileSearchTool(type="file_search") - ) + standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search")) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, @@ -140,9 +120,7 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): usage = Usage(server_tool_use={"web_search_requests": 1}) assert isinstance(usage.server_tool_use, ServerToolUse) - assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( - response_object=None, usage=usage - ) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage) def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): @@ -181,9 +159,7 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_serve standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] assert cost == per_query_cost * web_search_requests assert cost > 0.0 assert getattr(usage, "server_tool_use", None) is None @@ -221,9 +197,7 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] assert cost == per_query_cost * web_search_requests @@ -287,18 +261,14 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize( - "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] -) +@pytest.mark.parametrize("model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"]) def test_get_cost_for_gemini_web_search(model): """ Test that the cost for a web search is 0.00 when no response object is provided """ from litellm.types.utils import PromptTokensDetailsWrapper, Usage - usage = Usage( - prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) - ) + usage = Usage(prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1)) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, usage=usage, @@ -356,9 +326,7 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par ) assert web_search_cost > 0, "Web search cost should be non-zero" - assert ( - cost >= web_search_cost - ), f"completion_cost ({cost}) should include web search cost ({web_search_cost})" + assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})" @pytest.mark.parametrize( @@ -385,18 +353,14 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map): web_search_requests = 2 model_info = litellm.get_model_info(model) assert model_info["web_search_billing_unit"] == "per_query" - per_query_cost = model_info["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_query_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"] expected_cost = per_query_cost * web_search_requests usage = Usage( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=web_search_requests - ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=web_search_requests), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -408,8 +372,7 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map): ) assert cost == pytest.approx(expected_cost), ( - f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} " - f"per_query search fee, got ${cost}" + f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} per_query search fee, got ${cost}" ) @@ -452,17 +415,13 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map): model = "vertex_ai/gemini-2.5-flash" model_info = litellm.get_model_info(model) assert not model_info.get("web_search_billing_unit") - expected_cost = model_info["search_context_cost_per_query"][ - "search_context_size_medium" - ] + expected_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"] usage = Usage( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=2 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -474,8 +433,7 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map): ) assert cost == pytest.approx(expected_cost), ( - f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), " - f"got ${cost}" + f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), got ${cost}" ) @@ -501,9 +459,7 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=11, web_search_requests=2 - ), + prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -526,7 +482,6 @@ def _openai_responses_with_web_search_calls(model, num_calls): ResponseFunctionWebSearch, ) - output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -557,9 +512,7 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) for num_calls in (1, 3): @@ -586,9 +539,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ - "search_context_size_medium" - ] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] response = ResponsesAPIResponse.model_validate( { @@ -597,10 +548,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): "model": model, "object": "response", "status": "completed", - "output": [ - {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} - for i in range(3) - ], + "output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)], } ) assert all(isinstance(item, dict) for item in response.output) @@ -613,9 +561,7 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): standard_built_in_tools_params=None, ) - assert cost == pytest.approx(3 * per_call), ( - f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" - ) + assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" # Note: File search integration test removed due to complex annotation detection logic @@ -695,8 +641,6 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 - - def _openai_responses_response(model, output): return ResponsesAPIResponse.model_validate( { @@ -844,12 +788,8 @@ def test_completion_cost_includes_responses_image_generation_tool_cost(local_mod "size": "1024x1024", "result": "AAAA", } - response_with_image = _openai_responses_response( - "gpt-5", [image_item, dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)] - ) - response_without_image = _openai_responses_response( - "gpt-5", [dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)] - ) + response_with_image = _openai_responses_response("gpt-5", [image_item, dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)]) + response_without_image = _openai_responses_response("gpt-5", [dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)]) cost_with_image = litellm.completion_cost( completion_response=response_with_image, From dc4a6118ee06352f0897fe0a773b003ad1c70f66 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:32:52 +0000 Subject: [PATCH 03/10] test(cost): scope image generation tool cost tests to the new cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../test_tool_call_cost_tracking.py | 123 ++++++++++++------ 1 file changed, 83 insertions(+), 40 deletions(-) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index c3a5698bf69..1979229f0fc 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,3 +1,4 @@ + import pytest import litellm @@ -16,7 +17,9 @@ def test_web_search_cost_low(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_low"] + assert ( + cost == model_info["search_context_cost_per_query"]["search_context_size_low"] + ) def test_web_search_cost_medium(): @@ -27,7 +30,10 @@ def test_web_search_cost_medium(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_medium"] + assert ( + cost + == model_info["search_context_cost_per_query"]["search_context_size_medium"] + ) def test_web_search_cost_high(): @@ -38,21 +44,33 @@ def test_web_search_cost_high(): web_search_options=web_search_options, model_info=model_info ) - assert cost == model_info["search_context_cost_per_query"]["search_context_size_high"] + assert ( + cost == model_info["search_context_cost_per_query"]["search_context_size_high"] + ) # Test file search cost calculation def test_file_search_cost(): file_search = FileSearchTool(type="file_search") - cost = StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=file_search) + cost = StandardBuiltInToolCostTracking.get_cost_for_file_search( + file_search=file_search + ) assert cost == 0.0025 # $2.50/1000 calls = 0.0025 per call # Test edge cases def test_none_inputs(): # Test with None inputs - assert StandardBuiltInToolCostTracking.get_cost_for_web_search(web_search_options=None, model_info=None) == 0.0 - assert StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) == 0.0 + assert ( + StandardBuiltInToolCostTracking.get_cost_for_web_search( + web_search_options=None, model_info=None + ) + == 0.0 + ) + assert ( + StandardBuiltInToolCostTracking.get_cost_for_file_search(file_search=None) + == 0.0 + ) # Test the main get_cost_for_built_in_tools method @@ -77,7 +95,9 @@ def test_get_cost_for_built_in_tools_file_search(): Test that the cost for a file search is 0.00 when no response object is provided """ model = "gpt-4" - standard_built_in_tools_params = StandardBuiltInToolsParams(file_search=FileSearchTool(type="file_search")) + standard_built_in_tools_params = StandardBuiltInToolsParams( + file_search=FileSearchTool(type="file_search") + ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, @@ -120,7 +140,9 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict(): usage = Usage(server_tool_use={"web_search_requests": 1}) assert isinstance(usage.server_tool_use, ServerToolUse) - assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call(response_object=None, usage=usage) + assert StandardBuiltInToolCostTracking.response_object_includes_web_search_call( + response_object=None, usage=usage + ) def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use(): @@ -159,7 +181,9 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_serve standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] assert cost == per_query_cost * web_search_requests assert cost > 0.0 assert getattr(usage, "server_tool_use", None) is None @@ -197,7 +221,9 @@ def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none(): standard_built_in_tools_params=None, ) - per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] assert cost == per_query_cost * web_search_requests @@ -261,14 +287,18 @@ def test_anthropic_response_usage_block_preserves_server_tool_use(): assert dumped_usage["server_tool_use"] == {"web_search_requests": 2} -@pytest.mark.parametrize("model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"]) +@pytest.mark.parametrize( + "model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"] +) def test_get_cost_for_gemini_web_search(model): """ Test that the cost for a web search is 0.00 when no response object is provided """ from litellm.types.utils import PromptTokensDetailsWrapper, Usage - usage = Usage(prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1)) + usage = Usage( + prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=1) + ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( model=model, usage=usage, @@ -326,7 +356,9 @@ def test_completion_cost_includes_web_search_without_standard_built_in_tools_par ) assert web_search_cost > 0, "Web search cost should be non-zero" - assert cost >= web_search_cost, f"completion_cost ({cost}) should include web search cost ({web_search_cost})" + assert ( + cost >= web_search_cost + ), f"completion_cost ({cost}) should include web search cost ({web_search_cost})" @pytest.mark.parametrize( @@ -353,14 +385,18 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map): web_search_requests = 2 model_info = litellm.get_model_info(model) assert model_info["web_search_billing_unit"] == "per_query" - per_query_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"] + per_query_cost = model_info["search_context_cost_per_query"][ + "search_context_size_medium" + ] expected_cost = per_query_cost * web_search_requests usage = Usage( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=web_search_requests), + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=11, web_search_requests=web_search_requests + ), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -372,7 +408,8 @@ def test_gemini_3x_web_search_billed_per_query(model, local_model_cost_map): ) assert cost == pytest.approx(expected_cost), ( - f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} per_query search fee, got ${cost}" + f"Expected {web_search_requests} x ${per_query_cost} = ${expected_cost} " + f"per_query search fee, got ${cost}" ) @@ -415,13 +452,17 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map): model = "vertex_ai/gemini-2.5-flash" model_info = litellm.get_model_info(model) assert not model_info.get("web_search_billing_unit") - expected_cost = model_info["search_context_cost_per_query"]["search_context_size_medium"] + expected_cost = model_info["search_context_cost_per_query"][ + "search_context_size_medium" + ] usage = Usage( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2), + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=11, web_search_requests=2 + ), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -433,7 +474,8 @@ def test_gemini_2x_web_search_still_billed_per_prompt(local_model_cost_map): ) assert cost == pytest.approx(expected_cost), ( - f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), got ${cost}" + f"Expected flat ${expected_cost} per_prompt search fee (2 queries clamped to 1), " + f"got ${cost}" ) @@ -459,7 +501,9 @@ def test_web_search_provider_prefix_fallback_does_not_misprice_non_gemini_model( prompt_tokens=11, completion_tokens=100, total_tokens=111, - prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=11, web_search_requests=2), + prompt_tokens_details=PromptTokensDetailsWrapper( + text_tokens=11, web_search_requests=2 + ), ) cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( @@ -482,6 +526,7 @@ def _openai_responses_with_web_search_calls(model, num_calls): ResponseFunctionWebSearch, ) + output = [ ResponseFunctionWebSearch( id=f"ws_{i}", @@ -512,7 +557,9 @@ def test_openai_responses_web_search_multiplied_by_call_count(local_model_cost_m from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] usage = Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15) for num_calls in (1, 3): @@ -539,7 +586,9 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): from litellm.types.utils import Usage model = "gpt-4o-search-preview" - per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + per_call = litellm.get_model_info(model)["search_context_cost_per_query"][ + "search_context_size_medium" + ] response = ResponsesAPIResponse.model_validate( { @@ -548,7 +597,10 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): "model": model, "object": "response", "status": "completed", - "output": [{"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} for i in range(3)], + "output": [ + {"type": "web_search_call", "id": f"ws_{i}", "status": "completed"} + for i in range(3) + ], } ) assert all(isinstance(item, dict) for item in response.output) @@ -561,7 +613,9 @@ def test_web_search_call_count_reads_dict_output_items(local_model_cost_map): standard_built_in_tools_params=None, ) - assert cost == pytest.approx(3 * per_call), f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" + assert cost == pytest.approx(3 * per_call), ( + f"3 dict-shaped web searches must bill 3 x ${per_call}, got ${cost}" + ) # Note: File search integration test removed due to complex annotation detection logic @@ -641,6 +695,8 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = ( _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 + + def _openai_responses_response(model, output): return ResponsesAPIResponse.model_validate( { @@ -667,12 +723,7 @@ _GPT_IMAGE_1_HIGH_1024_COST_KEY = "high/1024-x-1024/gpt-image-1" def test_responses_image_generation_call_billed_as_tool_usage_cost(local_model_cost_map): - """ - Regression: a Responses API output carrying an image_generation_call item was - charged $0 of tool usage because get_cost_for_built_in_tools only looked for - web/file search calls. A completed image_generation_call must be billed at the - gpt-image-1 rate for its reported quality and size. - """ + """A completed image_generation_call in the Responses output bills at the gpt-image-1 rate for its quality/size.""" expected_image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] response = _openai_responses_response( "gpt-5", @@ -702,11 +753,7 @@ def test_responses_image_generation_call_billed_as_tool_usage_cost(local_model_c def test_responses_web_search_and_image_generation_costs_are_additive(local_model_cost_map): - """ - Regression: get_cost_for_built_in_tools returned early after the web search - branch, so a response billed for web search never reached image generation - pricing. A response with both must bill both. - """ + """A response billed for web search must still also bill its image_generation_call items.""" model = "gpt-4o-search-preview" image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] image_item = { @@ -775,11 +822,7 @@ def test_responses_incomplete_image_generation_call_not_billed(local_model_cost_ def test_completion_cost_includes_responses_image_generation_tool_cost(local_model_cost_map): - """ - An image_generation_call in the Responses output must flow through - completion_cost: the billed total for the same response without the image - item is the token-only baseline the image item must exceed by its tool cost. - """ + """The image tool fee must flow through completion_cost on top of the token-only baseline.""" image_item = { "type": "image_generation_call", "id": "ig_1", From f7db106bc84c64f16875dfb7bc221fb6f859ae8b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:33:51 +0000 Subject: [PATCH 04/10] refactor(cost): drop unused image generation call predicate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 1d4576a1511..e7fa3454411 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -235,15 +235,6 @@ class StandardBuiltInToolCostTracking: ) return max(count, 1) - @staticmethod - def response_object_includes_image_generation_call(response_object: object) -> bool: - """Check if the response object includes an image generation call (Responses API).""" - if not isinstance(response_object, ResponsesAPIResponse): - return False - return StandardBuiltInToolCostTracking.response_includes_output_type( - response_object=response_object, output_type="image_generation_call" - ) - @staticmethod def _image_generation_call_cost(output_item: object, custom_llm_provider: str | None) -> float: from litellm.cost_calculator import ( From 42c27d11dc68c486e293be7cb120d4db7c6cd8d6 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:39:18 +0000 Subject: [PATCH 05/10] style(cost): noqa the cast import for image generation tool cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index e7fa3454411..c5bfcaba0d5 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -3,7 +3,11 @@ Helper utilities for tracking the cost of built-in tools. """ from collections.abc import Mapping -from typing import Final, Literal, cast +from typing import ( + Final, + Literal, + cast, # noqa: TID251 # narrows SDK-union output items and dict fallbacks into typed views +) from pydantic import ValidationError From bb661f8305212ffab213bb4ff4d1250afab13845 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:43:11 +0000 Subject: [PATCH 06/10] style(cost): mark cast call sites for type discipline gate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index c5bfcaba0d5..ca9f8a0254b 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -37,7 +37,7 @@ from litellm.types.utils import ( def _output_item_field(output_item: object, field: str) -> object: if isinstance(output_item, dict): - fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) + fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) # cast-ok: narrowed by isinstance return fields.get(field) return getattr(output_item, field, None) @@ -266,7 +266,7 @@ class StandardBuiltInToolCostTracking: def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float: if not isinstance(response_object, ResponsesAPIResponse): return 0.0 - output: Final[list[object]] = cast(list[object], response_object.output) + output: Final[list[object]] = cast(list[object], response_object.output) # cast-ok: narrowed by isinstance return sum( StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, custom_llm_provider) for output_item in output From c5ee5432dd52c0146ed854933a8f370c4416f808 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 01:49:35 +0000 Subject: [PATCH 07/10] fix(cost): bill Responses API tool usage from usage.tool_usage and the image tool model Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 84 ++++++++-- litellm/types/llms/openai.py | 18 +++ .../test_tool_call_cost_tracking.py | 143 +++++++++++++++++- 3 files changed, 232 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index ca9f8a0254b..83cef0d49c3 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -47,15 +47,24 @@ def _output_item_type(output_item: object) -> str | None: return item_type if isinstance(item_type, str) else None -def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None: - tool_usage: Final = getattr(response_object, "tool_usage", None) - if tool_usage is None: +def _responses_tool_usage(response_object: ResponsesAPIResponse) -> ResponsesToolUsage | None: + top_level: Final = getattr(response_object, "tool_usage", None) + raw: Final = ( + top_level if top_level is not None else getattr(getattr(response_object, "usage", None), "tool_usage", None) + ) + if raw is None: return None try: - web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search + return ResponsesToolUsage.model_validate(raw) except ValidationError: return None - return None if web_search is None else web_search.num_requests + + +def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None: + tool_usage: Final = _responses_tool_usage(response_object) + if tool_usage is None: + return None + return None if tool_usage.web_search is None else tool_usage.web_search.num_requests def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool: @@ -240,7 +249,19 @@ class StandardBuiltInToolCostTracking: return max(count, 1) @staticmethod - def _image_generation_call_cost(output_item: object, custom_llm_provider: str | None) -> float: + def _image_generation_tool_model(response_object: ResponsesAPIResponse) -> str: + tools: Final[list[object]] = cast( + list[object], getattr(response_object, "tools", None) or [] + ) # cast-ok: tools entries may be dicts or pydantic objects + for tool in tools: + if _output_item_field(tool, "type") != "image_generation": + continue + if isinstance(model := _output_item_field(tool, "model"), str) and model: + return model + return "gpt-image-1" + + @staticmethod + def _image_generation_call_cost(output_item: object, tool_model: str, custom_llm_provider: str | None) -> float: from litellm.cost_calculator import ( default_image_cost_calculator, # pyright: ignore[reportUnknownVariableType] # optional_params param is untyped ) @@ -252,7 +273,7 @@ class StandardBuiltInToolCostTracking: size: Final = _output_item_field(output_item, "size") try: return default_image_cost_calculator( - model="gpt-image-1", + model=tool_model, custom_llm_provider=custom_llm_provider or "openai", quality=quality if isinstance(quality, str) and quality != "auto" else None, n=1, @@ -262,15 +283,53 @@ class StandardBuiltInToolCostTracking: verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e) return 0.0 + @staticmethod + def _image_generation_token_cost( + response_object: ResponsesAPIResponse, tool_model: str, custom_llm_provider: str | None + ) -> float | None: + tool_usage: Final = _responses_tool_usage(response_object) + if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0: + return None + try: + model_info: Final = litellm.get_model_info( + model=tool_model, custom_llm_provider=custom_llm_provider or "openai" + ) + except Exception as e: + verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e) + return 0.0 + image_gen: Final = tool_usage.image_gen + input_details: Final = image_gen.input_tokens_details + output_details: Final = image_gen.output_tokens_details + return ( + (input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0) + + (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0) + + (output_details.image_tokens if output_details else 0) + * (model_info.get("output_cost_per_image_token") or 0) + + (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0) + ) + @staticmethod def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float: if not isinstance(response_object, ResponsesAPIResponse): return 0.0 output: Final[list[object]] = cast(list[object], response_object.output) # cast-ok: narrowed by isinstance - return sum( - StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, custom_llm_provider) + completed_items: Final = tuple( + output_item for output_item in output if _output_item_type(output_item) == "image_generation_call" + and _output_item_field(output_item, "status") == "completed" + ) + if not completed_items: + return 0.0 + tool_model: Final = StandardBuiltInToolCostTracking._image_generation_tool_model(response_object) + token_cost: Final = StandardBuiltInToolCostTracking._image_generation_token_cost( + response_object, tool_model, custom_llm_provider + ) + if token_cost is not None: + return token_cost + return sum( + StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, tool_model, custom_llm_provider) + for output_item in completed_items ) @staticmethod @@ -508,9 +567,12 @@ class StandardBuiltInToolCostTracking: return False elif isinstance(response_object, ResponsesAPIResponse): # response api explicitly includes web_search_call in the output - return StandardBuiltInToolCostTracking.response_includes_output_type( + if StandardBuiltInToolCostTracking.response_includes_output_type( response_object=response_object, output_type="web_search_call" - ) + ): + return True + reported: Final = _reported_web_search_requests(response_object) + return isinstance(reported, int) and reported > 0 elif usage is not None: if get_web_search_requests_from_usage(usage) is not None or ( hasattr(usage, "prompt_tokens_details") diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index e3eac9b9205..a8495317382 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1332,10 +1332,28 @@ class WebSearchToolUsage(BaseModel): num_requests: NonNegativeInt +class ImageGenTokenDetails(BaseModel): + model_config = ConfigDict(frozen=True) + + image_tokens: NonNegativeInt = 0 + text_tokens: NonNegativeInt = 0 + + +class ImageGenToolUsage(BaseModel): + model_config = ConfigDict(frozen=True) + + input_tokens: NonNegativeInt = 0 + output_tokens: NonNegativeInt = 0 + total_tokens: NonNegativeInt = 0 + input_tokens_details: ImageGenTokenDetails | None = None + output_tokens_details: ImageGenTokenDetails | None = None + + class ResponsesToolUsage(BaseModel): model_config = ConfigDict(frozen=True) web_search: WebSearchToolUsage | None = None + image_gen: ImageGenToolUsage | None = None ResponsesAPIStatus = Literal["completed", "failed", "in_progress", "cancelled", "queued", "incomplete"] diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 1979229f0fc..570b701450d 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -697,7 +697,7 @@ _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012 -def _openai_responses_response(model, output): +def _openai_responses_response(model, output, usage=None, tools=None): return ResponsesAPIResponse.model_validate( { "id": "resp_1", @@ -706,7 +706,8 @@ def _openai_responses_response(model, output): "object": "response", "status": "completed", "output": output, - "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + "usage": usage or {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + **({"tools": tools} if tools is not None else {}), } ) @@ -851,3 +852,141 @@ def test_completion_cost_includes_responses_image_generation_tool_cost(local_mod assert cost_with_image - cost_without_image == pytest.approx( litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"] ) + + +def test_responses_usage_tool_usage_web_search_billed_without_output_item(local_model_cost_map): + """usage.tool_usage.web_search.num_requests bills web search even when no web_search_call item is present.""" + model = "gpt-5.4-mini" + per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"] + + for num_requests in (1, 2): + response = _openai_responses_response( + model, + [dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)], + usage={ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "tool_usage": {"web_search": {"num_requests": num_requests}}, + }, + ) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=model, + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + assert cost == pytest.approx(num_requests * per_call) + + +def _image_gen_token_usage(input_text, input_image, output_image, output_text): + return { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "tool_usage": { + "image_gen": { + "input_tokens": input_text + input_image, + "output_tokens": output_image + output_text, + "total_tokens": input_text + input_image + output_image + output_text, + "input_tokens_details": {"image_tokens": input_image, "text_tokens": input_text}, + "output_tokens_details": {"image_tokens": output_image, "text_tokens": output_text}, + } + }, + } + + +def _expected_image_gen_token_cost(model_info, input_text, input_image, output_image, output_text): + return ( + input_text * (model_info.get("input_cost_per_token") or 0) + + input_image * (model_info.get("input_cost_per_image_token") or 0) + + output_image * (model_info.get("output_cost_per_image_token") or 0) + + output_text * (model_info.get("output_cost_per_token") or 0) + ) + + +def test_responses_image_tool_model_from_tools_bills_token_usage(local_model_cost_map): + """The image tool's tools[].model is used and usage.tool_usage.image_gen tokens bill at that model's rates.""" + tool_model = "gpt-image-2" + model_info = litellm.get_model_info(tool_model, custom_llm_provider="openai") + tools = [{"type": "image_generation", "model": tool_model, "quality": "low", "size": "1024x1024"}] + output = [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": "low", + "size": "1024x1024", + "result": "AAAA", + }, + dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM), + ] + + response = _openai_responses_response("gpt-5", output, usage=_image_gen_token_usage(10, 0, 50, 0), tools=tools) + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + assert cost > 0 + assert cost == pytest.approx(_expected_image_gen_token_cost(model_info, 10, 0, 50, 0)) + + response_more_tokens = _openai_responses_response( + "gpt-5", output, usage=_image_gen_token_usage(10, 0, 80, 0), tools=tools + ) + cost_more = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response_more_tokens, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + assert cost_more != cost + assert cost_more == pytest.approx(_expected_image_gen_token_cost(model_info, 10, 0, 80, 0)) + + +def test_responses_zero_image_gen_tokens_fall_back_to_per_image_pricing(local_model_cost_map): + """An all-zero image_gen usage block keeps the per-image path for the tool's model/quality/size.""" + tool_model = "gpt-image-1" + quality = "low" + size = "1024x1024" + tools = [{"type": "image_generation", "model": tool_model, "quality": quality, "size": size}] + response = _openai_responses_response( + "gpt-5", + [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": quality, + "size": size, + "result": "AAAA", + } + ], + usage=_image_gen_token_usage(0, 0, 0, 0), + tools=tools, + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + from litellm.cost_calculator import default_image_cost_calculator + + assert cost == pytest.approx( + default_image_cost_calculator( + model=tool_model, + custom_llm_provider="openai", + quality=quality, + n=1, + size=size, + ) + ) + assert cost > 0 From cc6044aacc69d16bfeb2650205582706c07f505e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 01:54:16 +0000 Subject: [PATCH 08/10] style(cost): suppress BLE001 on image pricing fallbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 83cef0d49c3..164ba14b200 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -279,7 +279,7 @@ class StandardBuiltInToolCostTracking: n=1, size=size if isinstance(size, str) else None, ) - except Exception as e: + except Exception as e: # noqa: BLE001 # pricing helpers raise bare Exception for unmapped models; bill 0.0 verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e) return 0.0 @@ -294,7 +294,7 @@ class StandardBuiltInToolCostTracking: model_info: Final = litellm.get_model_info( model=tool_model, custom_llm_provider=custom_llm_provider or "openai" ) - except Exception as e: + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; bill 0.0 verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e) return 0.0 image_gen: Final = tool_usage.image_gen From 9cf8ef4cec6569a86ea3c5f97ed2de7c227b8e7e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 02:13:29 +0000 Subject: [PATCH 09/10] fix(cost): bill auto image size at the default and fall back when image tokens are unpriceable Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 9 +- .../test_tool_call_cost_tracking.py | 86 +++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 164ba14b200..1e5bab7ea3c 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -277,7 +277,7 @@ class StandardBuiltInToolCostTracking: custom_llm_provider=custom_llm_provider or "openai", quality=quality if isinstance(quality, str) and quality != "auto" else None, n=1, - size=size if isinstance(size, str) else None, + size=size if isinstance(size, str) and size != "auto" else None, ) except Exception as e: # noqa: BLE001 # pricing helpers raise bare Exception for unmapped models; bill 0.0 verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e) @@ -294,19 +294,20 @@ class StandardBuiltInToolCostTracking: model_info: Final = litellm.get_model_info( model=tool_model, custom_llm_provider=custom_llm_provider or "openai" ) - except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; bill 0.0 + except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e) - return 0.0 + return None image_gen: Final = tool_usage.image_gen input_details: Final = image_gen.input_tokens_details output_details: Final = image_gen.output_tokens_details - return ( + token_cost: Final = ( (input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0) + (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0) + (output_details.image_tokens if output_details else 0) * (model_info.get("output_cost_per_image_token") or 0) + (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0) ) + return token_cost if token_cost > 0 else None @staticmethod def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float: diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index 570b701450d..6d525bca654 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -990,3 +990,89 @@ def test_responses_zero_image_gen_tokens_fall_back_to_per_image_pricing(local_mo ) ) assert cost > 0 + + +def test_responses_auto_size_image_generation_call_billed_at_default_size(local_model_cost_map): + """An image_generation_call with size "auto" bills at the default size instead of erroring to $0.""" + from litellm.cost_calculator import default_image_cost_calculator + + response = _openai_responses_response( + "gpt-5", + [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": "high", + "size": "auto", + "result": "AAAA", + } + ], + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx( + default_image_cost_calculator( + model="gpt-image-1", + custom_llm_provider="openai", + quality="high", + n=1, + size=None, + ) + ) + assert cost > 0 + + +def test_responses_image_gen_total_without_token_details_falls_back_to_per_image(local_model_cost_map): + """A positive image_gen total with no token details falls back to per-image pricing, not $0.""" + tool_model = "gpt-image-1" + quality = "low" + size = "1024x1024" + response = _openai_responses_response( + "gpt-5", + [ + { + "type": "image_generation_call", + "id": "ig_1", + "status": "completed", + "quality": quality, + "size": size, + "result": "AAAA", + } + ], + usage={ + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "tool_usage": {"image_gen": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10}}, + }, + tools=[{"type": "image_generation", "model": tool_model, "quality": quality, "size": size}], + ) + + cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model="gpt-5", + response_object=response, + usage=None, + custom_llm_provider="openai", + standard_built_in_tools_params=None, + ) + + from litellm.cost_calculator import default_image_cost_calculator + + assert cost == pytest.approx( + default_image_cost_calculator( + model=tool_model, + custom_llm_provider="openai", + quality=quality, + n=1, + size=size, + ) + ) + assert cost > 0 From 0761288974837e3949bfae700ee0f2561f0aa98f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 18:28:17 +0000 Subject: [PATCH 10/10] style(cost): drop unused cast-ok marker for LIT013 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 1e5bab7ea3c..8896b50ea32 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -250,9 +250,7 @@ class StandardBuiltInToolCostTracking: @staticmethod def _image_generation_tool_model(response_object: ResponsesAPIResponse) -> str: - tools: Final[list[object]] = cast( - list[object], getattr(response_object, "tools", None) or [] - ) # cast-ok: tools entries may be dicts or pydantic objects + tools: Final[list[object]] = cast(list[object], getattr(response_object, "tools", None) or []) for tool in tools: if _output_item_field(tool, "type") != "image_generation": continue