scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(cost): preserve Anthropic server_tool_use web search usage in cost tracking (#31355)

Browse source 
* fix(cost): preserve Anthropic server_tool_use web search usage in cost tracking

Anthropic /v1/messages responses report built-in web search usage under
usage.server_tool_use.web_search_requests, but the sync cost path reconstructs
an OpenAI-shape Usage that drops server_tool_use and validates the response
through AnthropicResponse, which previously stripped the field. Either path
could leave the web-search fee uncounted.

AnthropicResponseUsageBlock now allows extra fields so model_validate/model_dump
keeps server_tool_use, and the built-in tool cost tracker reads the web search
count straight off the raw Anthropic response dict when the reconstructed Usage
lacks it, synthesizing a ServerToolUse without mutating the caller's Usage.

* fix(lint): use PEP 604 unions in anthropic web search probes to satisfy strict-rule budget

* refactor(cost): move Anthropic web search response parsing into llms/anthropic

Relocate the raw /v1/messages web-search-count probe out of the shared
built-in tool cost tracker into litellm/llms/anthropic/cost_calculation.py,
next to get_cost_for_anthropic_web_search, so provider-specific response
parsing lives under llms/. The core cost tracker now delegates to
get_anthropic_web_search_requests_from_response and keeps only the generic
Usage/ServerToolUse orchestration.

* fix(cost): price Anthropic web search when only the raw response carries the count

response_object_includes_web_search_call enters the web search branch as
soon as the raw Anthropic dict reports usage.server_tool_use.web_search_requests,
but _usage_with_anthropic_web_search bailed when the caller did not also pass
a Usage object. _handle_web_search_cost then skipped the per-request anthropic
path and fell back to the flat search_context_size_medium tier, charging a
fixed fee instead of per_query x count (or zero when the count is zero).

Synthesize a Usage from the raw dict when no Usage is supplied so count-based
pricing runs uniformly regardless of how the response reaches the tracker.

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
Mateo Wang 2026-06-26 20:36:56 -07:00 • committed by GitHub
parent e4aedb0342
commit ef66620223
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 219 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
View file

@ -17,6 +17,7 @@ from litellm.types.utils import (
ModelInfo,
ModelResponse,
SearchContextCostPerQuery,
ServerToolUse,
StandardBuiltInToolsParams,
Usage,
)
@ -58,6 +59,7 @@ class StandardBuiltInToolCostTracking:
custom_llm_provider=custom_llm_provider,
usage=usage,
standard_built_in_tools_params=standard_built_in_tools_params,
response_object=response_object,
)
# Handle file search
@ -83,6 +85,7 @@ class StandardBuiltInToolCostTracking:
custom_llm_provider: Optional[str],
usage: Optional[Usage],
standard_built_in_tools_params: StandardBuiltInToolsParams,
response_object: object = None,
) -> float:
"""Handle web search cost calculation."""
from litellm.llms import get_cost_for_web_search_request
@ -105,14 +108,20 @@ class StandardBuiltInToolCostTracking:
if custom_llm_provider is None and model_info is not None:
custom_llm_provider = model_info["litellm_provider"]
resolved_usage = (
StandardBuiltInToolCostTracking._usage_with_anthropic_web_search(
usage=usage, response_object=response_object
)
)
if (
model_info is not None
and usage is not None
and resolved_usage is not None
and custom_llm_provider is not None
):
result = get_cost_for_web_search_request(
custom_llm_provider=custom_llm_provider,
usage=usage,
usage=resolved_usage,
model_info=model_info,
)
if result is not None:
@ -312,6 +321,33 @@ class StandardBuiltInToolCostTracking:
return None
return None
@staticmethod
def _usage_with_anthropic_web_search(
usage: Usage | None, response_object: object
) -> Usage | None:
"""Return a Usage carrying server_tool_use.web_search_requests sourced from a
raw Anthropic /v1/messages response dict when the reconstructed Usage dropped
it (or was never supplied). The original Usage is returned unchanged when it
already exposes the field or the response is not an Anthropic dict."""
from litellm.llms.anthropic.cost_calculation import (
get_anthropic_web_search_requests_from_response,
)
if usage is not None and (
_get_web_search_requests(getattr(usage, "server_tool_use", None))
is not None
):
return usage
web_search_requests = get_anthropic_web_search_requests_from_response(
response_object
)
if web_search_requests is None:
return usage
server_tool_use = ServerToolUse(web_search_requests=web_search_requests)
if usage is None:
return Usage(server_tool_use=server_tool_use)
return usage.model_copy(update={"server_tool_use": server_tool_use})
@staticmethod
def response_object_includes_web_search_call(
response_object: Any, usage: Optional[Usage] = None
@ -322,9 +358,16 @@ class StandardBuiltInToolCostTracking:
This covers:
- Chat Completion Response (ModelResponse)
- ResponsesAPIResponse (streaming + non-streaming)
- Anthropic /v1/messages raw response dict
"""
from litellm.llms.anthropic.cost_calculation import (
get_anthropic_web_search_requests_from_response,
)
from litellm.types.utils import PromptTokensDetailsWrapper
if get_anthropic_web_search_requests_from_response(response_object) is not None:
return True
if isinstance(response_object, ModelResponse):
# chat completions only include url_citation annotations when a web search call is made
has_url_citations = (

30
litellm/llms/anthropic/cost_calculation.py
View file

@ -5,6 +5,8 @@ Helper util for handling anthropic-specific cost calculation
from typing import TYPE_CHECKING, Optional, Tuple
from pydantic import BaseModel, ValidationError
from litellm.litellm_core_utils.llm_cost_calc.utils import (
_get_token_base_cost,
_get_web_search_requests,
@ -111,6 +113,34 @@ def cost_per_token(
return prompt_cost, completion_cost
class _AnthropicServerToolUseProbe(BaseModel):
web_search_requests: int | None = None
class _AnthropicUsageProbe(BaseModel):
server_tool_use: _AnthropicServerToolUseProbe | None = None
class _AnthropicResponseProbe(BaseModel):
usage: _AnthropicUsageProbe | None = None
def get_anthropic_web_search_requests_from_response(
response_object: object,
) -> int | None:
"""Read usage.server_tool_use.web_search_requests from a raw Anthropic
/v1/messages response dict, returning None when absent."""
if not isinstance(response_object, dict):
return None
try:
probe = _AnthropicResponseProbe.model_validate(response_object)
except ValidationError:
return None
if probe.usage is None or probe.usage.server_tool_use is None:
return None
return probe.usage.server_tool_use.web_search_requests
def get_cost_for_anthropic_web_search(
model_info: Optional["ModelInfo"] = None,
usage: Optional["Usage"] = None,

2
litellm/types/llms/anthropic.py
View file

@ -625,6 +625,8 @@ class AnthropicResponseContentBlockRedactedThinking(BaseModel):
class AnthropicResponseUsageBlock(BaseModel):
model_config = ConfigDict(extra="allow")
input_tokens: int
output_tokens: int

142
tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
View file

@ -158,6 +158,148 @@ def test_get_cost_for_anthropic_web_search_with_server_tool_use_dict():
)
def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_drops_server_tool_use():
"""
Regression: on the Anthropic /v1/messages sync cost path the response is the raw
Anthropic dict while the reconstructed OpenAI-shape Usage drops server_tool_use.
The web-search fee must still be charged by reading the count off the raw dict,
and the passed-in Usage must not be mutated.
"""
from litellm.types.utils import Usage
model = "claude-3-7-sonnet-20250219"
web_search_requests = 3
raw_response = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {
"input_tokens": 100,
"output_tokens": 50,
"server_tool_use": {"web_search_requests": web_search_requests},
},
}
usage = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
assert getattr(usage, "server_tool_use", None) is None
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
usage=usage,
response_object=raw_response,
custom_llm_provider="anthropic",
standard_built_in_tools_params=None,
)
per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][
"search_context_size_medium"
]
assert cost == per_query_cost * web_search_requests
assert cost > 0.0
assert getattr(usage, "server_tool_use", None) is None
def test_anthropic_web_search_cost_from_raw_response_dict_when_usage_is_none():
"""
Regression: when a caller hands the cost tracker a raw Anthropic dict without a
parallel Usage object, the web-search fee must still be priced per request from
usage.server_tool_use.web_search_requests on the dict instead of falling back to
the flat search_context_size_medium tier.
"""
model = "claude-3-7-sonnet-20250219"
web_search_requests = 4
raw_response = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {
"input_tokens": 100,
"output_tokens": 50,
"server_tool_use": {"web_search_requests": web_search_requests},
},
}
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
usage=None,
response_object=raw_response,
custom_llm_provider="anthropic",
standard_built_in_tools_params=None,
)
per_query_cost = litellm.get_model_info(model)["search_context_cost_per_query"][
"search_context_size_medium"
]
assert cost == per_query_cost * web_search_requests
def test_anthropic_web_search_zero_requests_from_raw_response_charges_zero():
"""
Regression: a raw Anthropic dict reporting zero web search requests must price
the call at zero rather than charging the default medium-tier fee.
"""
model = "claude-3-7-sonnet-20250219"
raw_response = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": model,
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {
"input_tokens": 100,
"output_tokens": 50,
"server_tool_use": {"web_search_requests": 0},
},
}
cost = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
model=model,
usage=None,
response_object=raw_response,
custom_llm_provider="anthropic",
standard_built_in_tools_params=None,
)
assert cost == 0.0
def test_anthropic_response_usage_block_preserves_server_tool_use():
"""
Regression: AnthropicResponse.model_validate(...).model_dump() must keep
server_tool_use so the /v1/messages logging fallback does not strip the
web-search usage before cost tracking sees it.
"""
from litellm.types.llms.anthropic import AnthropicResponse
raw_response = {
"id": "msg_1",
"type": "message",
"role": "assistant",
"model": "claude-3-7-sonnet-20250219",
"content": [{"type": "text", "text": "hi"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {
"input_tokens": 100,
"output_tokens": 50,
"server_tool_use": {"web_search_requests": 2},
},
}
dumped_usage = AnthropicResponse.model_validate(raw_response).model_dump()["usage"]
assert dumped_usage["server_tool_use"] == {"web_search_requests": 2}
@pytest.mark.parametrize(
"model", ["gemini/gemini-2.0-flash-001", "gemini-2.0-flash-001"]
)