scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

Merge 0761288974 into dd31692282

Browse source
This commit is contained in:
devin-ai-integration[bot] 2026-10-03 16:30:34 -04:00 • committed by GitHub
commit a67e5d8e54
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 546 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

170
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -3,11 +3,16 @@ Helper utilities for tracking the cost of built-in tools.
"""
from collections.abc import Mapping
from typing import Final, Literal
from typing import (
Final,
Literal,
cast, # noqa: TID251 # narrows SDK-union output items and dict fallbacks into typed views
)
from pydantic import ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
from litellm.litellm_core_utils.llm_cost_calc.utils import (
get_web_search_requests_from_usage,
@ -30,20 +35,36 @@ from litellm.types.utils import (
)
def _output_item_field(output_item: object, field: str) -> object:
if isinstance(output_item, dict):
fields: Final[Mapping[str, object]] = cast(Mapping[str, object], output_item) # cast-ok: narrowed by isinstance
return fields.get(field)
return getattr(output_item, field, None)
def _output_item_type(output_item: object) -> str | None:
item_type: Final = output_item.get("type") if isinstance(output_item, dict) else getattr(output_item, "type", None)
item_type: Final = _output_item_field(output_item, "type")
return item_type if isinstance(item_type, str) else None
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = getattr(response_object, "tool_usage", None)
if tool_usage is None:
def _responses_tool_usage(response_object: ResponsesAPIResponse) -> ResponsesToolUsage | None:
top_level: Final = getattr(response_object, "tool_usage", None)
raw: Final = (
top_level if top_level is not None else getattr(getattr(response_object, "usage", None), "tool_usage", None)
)
if raw is None:
return None
try:
web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
return ResponsesToolUsage.model_validate(raw)
except ValidationError:
return None
return None if web_search is None else web_search.num_requests
def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None:
return None
return None if tool_usage.web_search is None else tool_usage.web_search.num_requests
def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
@ -87,31 +108,48 @@ class StandardBuiltInToolCostTracking:
usage=usage,
)
image_generation_cost: Final = StandardBuiltInToolCostTracking._handle_image_generation_cost(
response_object=response_object,
custom_llm_provider=custom_llm_provider,
)
# Handle web search
if StandardBuiltInToolCostTracking.response_object_includes_web_search_call(
response_object=response_object, usage=usage
):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_web_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
)
)
# Handle file search
if StandardBuiltInToolCostTracking.response_object_includes_file_search_call(response_object=response_object):
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_file_search_cost(
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_file_search_cost(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
)
# Handle Azure assistant features
return (
google_maps_grounding_cost
+ image_generation_cost
+ StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
# Handle Azure assistant features
return google_maps_grounding_cost + StandardBuiltInToolCostTracking._handle_azure_assistant_costs(
model=model,
custom_llm_provider=custom_llm_provider,
standard_built_in_tools_params=standard_built_in_tools_params,
)
@staticmethod
@ -210,6 +248,89 @@ class StandardBuiltInToolCostTracking:
)
return max(count, 1)
@staticmethod
def _image_generation_tool_model(response_object: ResponsesAPIResponse) -> str:
tools: Final[list[object]] = cast(list[object], getattr(response_object, "tools", None) or [])
for tool in tools:
if _output_item_field(tool, "type") != "image_generation":
continue
if isinstance(model := _output_item_field(tool, "model"), str) and model:
return model
return "gpt-image-1"
@staticmethod
def _image_generation_call_cost(output_item: object, tool_model: str, custom_llm_provider: str | None) -> float:
from litellm.cost_calculator import (
default_image_cost_calculator, # pyright: ignore[reportUnknownVariableType] # optional_params param is untyped
)
status: Final = _output_item_field(output_item, "status")
if status != "completed":
return 0.0
quality: Final = _output_item_field(output_item, "quality")
size: Final = _output_item_field(output_item, "size")
try:
return default_image_cost_calculator(
model=tool_model,
custom_llm_provider=custom_llm_provider or "openai",
quality=quality if isinstance(quality, str) and quality != "auto" else None,
n=1,
size=size if isinstance(size, str) and size != "auto" else None,
)
except Exception as e: # noqa: BLE001 # pricing helpers raise bare Exception for unmapped models; bill 0.0
verbose_logger.debug("Could not price Responses API image_generation_call item: %s", e)
return 0.0
@staticmethod
def _image_generation_token_cost(
response_object: ResponsesAPIResponse, tool_model: str, custom_llm_provider: str | None
) -> float | None:
tool_usage: Final = _responses_tool_usage(response_object)
if tool_usage is None or tool_usage.image_gen is None or tool_usage.image_gen.total_tokens <= 0:
return None
try:
model_info: Final = litellm.get_model_info(
model=tool_model, custom_llm_provider=custom_llm_provider or "openai"
)
except Exception as e: # noqa: BLE001 # get_model_info raises bare Exception for unmapped models; fall back
verbose_logger.debug("Could not resolve pricing for image tool model %s: %s", tool_model, e)
return None
image_gen: Final = tool_usage.image_gen
input_details: Final = image_gen.input_tokens_details
output_details: Final = image_gen.output_tokens_details
token_cost: Final = (
(input_details.text_tokens if input_details else 0) * (model_info.get("input_cost_per_token") or 0)
+ (input_details.image_tokens if input_details else 0) * (model_info.get("input_cost_per_image_token") or 0)
+ (output_details.image_tokens if output_details else 0)
* (model_info.get("output_cost_per_image_token") or 0)
+ (output_details.text_tokens if output_details else 0) * (model_info.get("output_cost_per_token") or 0)
)
return token_cost if token_cost > 0 else None
@staticmethod
def _handle_image_generation_cost(response_object: object, custom_llm_provider: str | None) -> float:
if not isinstance(response_object, ResponsesAPIResponse):
return 0.0
output: Final[list[object]] = cast(list[object], response_object.output) # cast-ok: narrowed by isinstance
completed_items: Final = tuple(
output_item
for output_item in output
if _output_item_type(output_item) == "image_generation_call"
and _output_item_field(output_item, "status") == "completed"
)
if not completed_items:
return 0.0
tool_model: Final = StandardBuiltInToolCostTracking._image_generation_tool_model(response_object)
token_cost: Final = StandardBuiltInToolCostTracking._image_generation_token_cost(
response_object, tool_model, custom_llm_provider
)
if token_cost is not None:
return token_cost
return sum(
StandardBuiltInToolCostTracking._image_generation_call_cost(output_item, tool_model, custom_llm_provider)
for output_item in completed_items
)
@staticmethod
def _handle_file_search_cost(
model: str,
@ -445,9 +566,12 @@ class StandardBuiltInToolCostTracking:
return False
elif isinstance(response_object, ResponsesAPIResponse):
# response api explicitly includes web_search_call in the output
return StandardBuiltInToolCostTracking.response_includes_output_type(
if StandardBuiltInToolCostTracking.response_includes_output_type(
response_object=response_object, output_type="web_search_call"
)
):
return True
reported: Final = _reported_web_search_requests(response_object)
return isinstance(reported, int) and reported > 0
elif usage is not None:
if get_web_search_requests_from_usage(usage) is not None or (
hasattr(usage, "prompt_tokens_details")
@ -505,7 +629,7 @@ class StandardBuiltInToolCostTracking:
@staticmethod
def response_includes_output_type(
response_object: ResponsesAPIResponse,
output_type: Literal["web_search_call", "file_search_call"],
output_type: Literal["web_search_call", "file_search_call", "image_generation_call"],
) -> bool:
"""
Check if the ResponsesAPIResponse includes one of the specified output types.

18
litellm/types/llms/openai.py
View file

@ -1372,10 +1372,28 @@ class WebSearchToolUsage(BaseModel):
num_requests: NonNegativeInt
class ImageGenTokenDetails(BaseModel):
model_config = ConfigDict(frozen=True)
image_tokens: NonNegativeInt = 0
text_tokens: NonNegativeInt = 0
class ImageGenToolUsage(BaseModel):
model_config = ConfigDict(frozen=True)
input_tokens: NonNegativeInt = 0
output_tokens: NonNegativeInt = 0
total_tokens: NonNegativeInt = 0
input_tokens_details: ImageGenTokenDetails | None = None
output_tokens_details: ImageGenTokenDetails | None = None
class ResponsesToolUsage(BaseModel):
model_config = ConfigDict(frozen=True)
web_search: WebSearchToolUsage | None = None
image_gen: ImageGenToolUsage | None = None
ResponsesAPIStatus = Literal["completed", "failed", "in_progress", "cancelled", "queued", "incomplete"]

381
tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
View file

@ -572,3 +572,384 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
def _openai_responses_response(model, output, usage=None, tools=None):
return ResponsesAPIResponse.model_validate(
{
"id": "resp_1",
"created_at": 1754900000,
"model": model,
"object": "response",
"status": "completed",
"output": output,
"usage": usage or {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
**({"tools": tools} if tools is not None else {}),
}
)
_ASSISTANT_MESSAGE_OUTPUT_ITEM = {
"type": "message",
"id": "msg_1",
"role": "assistant",
"status": "completed",
"content": [{"type": "output_text", "text": "done", "annotations": []}],
}
_GPT_IMAGE_1_HIGH_1024_COST_KEY = "high/1024-x-1024/gpt-image-1"
def test_responses_image_generation_call_billed_as_tool_usage_cost(local_model_cost_map):
"""A completed image_generation_call in the Responses output bills at the gpt-image-1 rate for its quality/size."""
expected_image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"]
response = _openai_responses_response(
"gpt-5",
[
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": "high",
"size": "1024x1024",
"result": "AAAA",
},
dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM),
],
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost > 0
assert cost == pytest.approx(expected_image_cost)
def test_responses_web_search_and_image_generation_costs_are_additive(local_model_cost_map):
"""A response billed for web search must still also bill its image_generation_call items."""
model = "gpt-4o-search-preview"
image_cost = litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"]
image_item = {
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": "high",
"size": "1024x1024",
"result": "AAAA",
}
web_search_item = {"type": "web_search_call", "id": "ws_1", "status": "completed"}
combined = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=_openai_responses_response(model, [web_search_item, image_item]),
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
web_search_only = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=_openai_responses_response(model, [web_search_item]),
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
image_only = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=_openai_responses_response(model, [image_item]),
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert image_only == pytest.approx(image_cost)
assert web_search_only > 0
assert combined == pytest.approx(web_search_only + image_only)
def test_responses_incomplete_image_generation_call_not_billed(local_model_cost_map):
"""A failed image_generation_call produced no billable image, so it must cost $0."""
response = _openai_responses_response(
"gpt-5",
[
{
"type": "image_generation_call",
"id": "ig_1",
"status": "failed",
"quality": "high",
"size": "1024x1024",
"result": None,
},
dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM),
],
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost == 0.0
def test_completion_cost_includes_responses_image_generation_tool_cost(local_model_cost_map):
"""The image tool fee must flow through completion_cost on top of the token-only baseline."""
image_item = {
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": "high",
"size": "1024x1024",
"result": "AAAA",
}
response_with_image = _openai_responses_response("gpt-5", [image_item, dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)])
response_without_image = _openai_responses_response("gpt-5", [dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)])
cost_with_image = litellm.completion_cost(
completion_response=response_with_image,
model="gpt-5",
custom_llm_provider="openai",
call_type="aresponses",
)
cost_without_image = litellm.completion_cost(
completion_response=response_without_image,
model="gpt-5",
custom_llm_provider="openai",
call_type="aresponses",
)
assert cost_with_image > cost_without_image
assert cost_with_image - cost_without_image == pytest.approx(
litellm.model_cost[_GPT_IMAGE_1_HIGH_1024_COST_KEY]["input_cost_per_image"]
)
def test_responses_usage_tool_usage_web_search_billed_without_output_item(local_model_cost_map):
"""usage.tool_usage.web_search.num_requests bills web search even when no web_search_call item is present."""
model = "gpt-5.4-mini"
per_call = litellm.get_model_info(model)["search_context_cost_per_query"]["search_context_size_medium"]
for num_requests in (1, 2):
response = _openai_responses_response(
model,
[dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM)],
usage={
"input_tokens": 10,
"output_tokens": 5,
"total_tokens": 15,
"tool_usage": {"web_search": {"num_requests": num_requests}},
},
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(num_requests * per_call)
def _image_gen_token_usage(input_text, input_image, output_image, output_text):
return {
"input_tokens": 10,
"output_tokens": 5,
"total_tokens": 15,
"tool_usage": {
"image_gen": {
"input_tokens": input_text + input_image,
"output_tokens": output_image + output_text,
"total_tokens": input_text + input_image + output_image + output_text,
"input_tokens_details": {"image_tokens": input_image, "text_tokens": input_text},
"output_tokens_details": {"image_tokens": output_image, "text_tokens": output_text},
}
},
}
def _expected_image_gen_token_cost(model_info, input_text, input_image, output_image, output_text):
return (
input_text * (model_info.get("input_cost_per_token") or 0)
+ input_image * (model_info.get("input_cost_per_image_token") or 0)
+ output_image * (model_info.get("output_cost_per_image_token") or 0)
+ output_text * (model_info.get("output_cost_per_token") or 0)
)
def test_responses_image_tool_model_from_tools_bills_token_usage(local_model_cost_map):
"""The image tool's tools[].model is used and usage.tool_usage.image_gen tokens bill at that model's rates."""
tool_model = "gpt-image-2"
model_info = litellm.get_model_info(tool_model, custom_llm_provider="openai")
tools = [{"type": "image_generation", "model": tool_model, "quality": "low", "size": "1024x1024"}]
output = [
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": "low",
"size": "1024x1024",
"result": "AAAA",
},
dict(_ASSISTANT_MESSAGE_OUTPUT_ITEM),
]
response = _openai_responses_response("gpt-5", output, usage=_image_gen_token_usage(10, 0, 50, 0), tools=tools)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost > 0
assert cost == pytest.approx(_expected_image_gen_token_cost(model_info, 10, 0, 50, 0))
response_more_tokens = _openai_responses_response(
"gpt-5", output, usage=_image_gen_token_usage(10, 0, 80, 0), tools=tools
)
cost_more = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response_more_tokens,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost_more != cost
assert cost_more == pytest.approx(_expected_image_gen_token_cost(model_info, 10, 0, 80, 0))
def test_responses_zero_image_gen_tokens_fall_back_to_per_image_pricing(local_model_cost_map):
"""An all-zero image_gen usage block keeps the per-image path for the tool's model/quality/size."""
tool_model = "gpt-image-1"
quality = "low"
size = "1024x1024"
tools = [{"type": "image_generation", "model": tool_model, "quality": quality, "size": size}]
response = _openai_responses_response(
"gpt-5",
[
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": quality,
"size": size,
"result": "AAAA",
}
],
usage=_image_gen_token_usage(0, 0, 0, 0),
tools=tools,
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
from litellm.cost_calculator import default_image_cost_calculator
assert cost == pytest.approx(
default_image_cost_calculator(
model=tool_model,
custom_llm_provider="openai",
quality=quality,
n=1,
size=size,
)
)
assert cost > 0
def test_responses_auto_size_image_generation_call_billed_at_default_size(local_model_cost_map):
"""An image_generation_call with size "auto" bills at the default size instead of erroring to $0."""
from litellm.cost_calculator import default_image_cost_calculator
response = _openai_responses_response(
"gpt-5",
[
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": "high",
"size": "auto",
"result": "AAAA",
}
],
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
assert cost == pytest.approx(
default_image_cost_calculator(
model="gpt-image-1",
custom_llm_provider="openai",
quality="high",
n=1,
size=None,
)
)
assert cost > 0
def test_responses_image_gen_total_without_token_details_falls_back_to_per_image(local_model_cost_map):
"""A positive image_gen total with no token details falls back to per-image pricing, not $0."""
tool_model = "gpt-image-1"
quality = "low"
size = "1024x1024"
response = _openai_responses_response(
"gpt-5",
[
{
"type": "image_generation_call",
"id": "ig_1",
"status": "completed",
"quality": quality,
"size": size,
"result": "AAAA",
}
],
usage={
"input_tokens": 10,
"output_tokens": 5,
"total_tokens": 15,
"tool_usage": {"image_gen": {"input_tokens": 5, "output_tokens": 5, "total_tokens": 10}},
},
tools=[{"type": "image_generation", "model": tool_model, "quality": quality, "size": size}],
)
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model="gpt-5",
response_object=response,
usage=None,
custom_llm_provider="openai",
standard_built_in_tools_params=None,
)
from litellm.cost_calculator import default_image_cost_calculator
assert cost == pytest.approx(
default_image_cost_calculator(
model=tool_model,
custom_llm_provider="openai",
quality=quality,
n=1,
size=size,
)
)
assert cost > 0