scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

feat(proxy): opt-in free-tier allowance for Google Search grounding spend (#47494)

Browse source 
* feat(proxy): opt-in free-tier allowance for Google Search grounding spend

* fix(proxy): declare the search free tier counter's service target

* test(cost_map): allow search_free_requests in the cost map schema

* fix(proxy): skip cache hits and never free more than the billed cost in the search free tier

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>

Backport of #47494 to stable/1.105.x.
Cherry-picked from e63aec029a (main).
This commit is contained in:
devin-ai-integration[bot] 2026-10-21 00:10:06 +00:00
parent fd5eec97b1
commit 7ffaaab9a0
14 changed files with 525 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
ci_cd/generate_model_prices_schema.py
View file

@ -91,6 +91,17 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
},
"additionalProperties": False,
},
"search_free_requests": {
"type": "object",
"description": "Free Google Search grounding requests per period, shared across models in the same shared_group.",
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
"properties": {
"count": NONNEG_INTEGER,
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": NON_BLANK_STRING,
},
},
"guardrail_cost_per_unit": {
"type": "object",
"description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",

1
litellm/__init__.py
View file

@ -562,6 +562,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:

1
litellm/constants.py
View file

@ -1896,6 +1896,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",

47
litellm/model_prices_and_context_window_backup.json
View file

@ -26878,6 +26878,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -26990,6 +26991,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27041,6 +27043,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27087,6 +27090,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27129,6 +27133,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27219,6 +27224,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27280,6 +27286,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27339,6 +27346,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27425,6 +27433,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27479,6 +27488,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27525,6 +27535,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27712,6 +27723,7 @@
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27779,6 +27791,7 @@
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27834,6 +27847,7 @@
"supports_web_search": true,
"supports_url_context": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27890,6 +27904,7 @@
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27940,6 +27955,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28001,6 +28017,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28064,6 +28081,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28123,6 +28141,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28182,6 +28201,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28328,6 +28348,7 @@
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28389,6 +28410,7 @@
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28716,6 +28738,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28760,6 +28783,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28821,6 +28845,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28970,6 +28995,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29080,6 +29106,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29170,6 +29197,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29301,6 +29329,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 250000,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29504,6 +29533,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 800000,
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29601,6 +29631,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29661,6 +29692,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29717,6 +29749,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29780,6 +29813,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29845,6 +29879,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29906,6 +29941,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29967,6 +30003,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30062,6 +30099,7 @@
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30125,6 +30163,7 @@
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30180,6 +30219,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30272,6 +30312,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30335,6 +30376,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30394,6 +30436,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30453,6 +30496,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50523,6 +50567,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50585,6 +50630,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50645,6 +50691,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,

30
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -1,9 +1,11 @@
import asyncio
import traceback
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from datetime import datetime, timezone
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from typing_extensions import TypeIs
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.batches.batch_utils import batch_cost_is_final
@ -30,6 +32,7 @@ from litellm.proxy.db.db_spend_update_writer import (
debitable_model_access_groups,
get_llm_router,
)
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route
from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope
from litellm.proxy.spend_tracking.spend_event import (
@ -106,6 +109,16 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter:
return proxy_logging_obj.db_spend_update_writer
def _is_str_keyed_mapping(
value: object,
) -> TypeIs[Mapping[str, object]]: # guard-ok: success-callback payloads are str-keyed dicts
return isinstance(value, Mapping)
def _str_keyed_fields(value: object) -> Mapping[str, object]:
return value if _is_str_keyed_mapping(value) else {}
def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]:
"""The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any."""
attempts: Final = request_data.get("billed_failed_attempts")
@ -434,8 +447,21 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists
response_cost=priced_cost,
completion_response=completion_response,
counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache,
now=datetime.now(timezone.utc),
)
if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None
else priced_cost
)
tags: Final = _get_request_tags_for_cost_tracking(
sl_object=sl_object,

121
litellm/proxy/hooks/search_free_tier.py Normal file
View file

@ -0,0 +1,121 @@
from collections.abc import MutableMapping
from datetime import datetime
from types import MappingProxyType
from typing import Final, Protocol
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage
_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"})
_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"})
_TTL_GRACE_SECONDS: Final = 3600
_COUNTED_KWARG: Final = "search_free_tier_counted"
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
class SharedCounter(Protocol):
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ...
class _UsageCarrier(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
usage: Usage | None = None
def freed_units(allowance: int, units: int, total_after: int) -> int:
"""Units of this request that landed under the allowance, given the shared total after they were added."""
return max(0, min(units, allowance - (total_after - units)))
def search_free_tier_key(allowance: SearchFreeRequests, now: datetime) -> str:
return f"litellm:search_free_tier:{allowance['shared_group']}:{now.strftime(_PERIOD_STAMP[allowance['period']])}"
def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int:
reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC")
return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS
def _usage_of(completion_response: object) -> Usage | None:
try:
return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage
except ValidationError:
return None
def _grounding_queries(usage: Usage) -> int:
details: Final = usage.prompt_tokens_details
from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None
return from_details or get_web_search_requests_from_usage(usage) or 0
def _billable_units(usage: Usage, model_info: ModelInfo) -> int:
queries: Final = _grounding_queries(usage)
if queries == 0:
return 0
return queries if model_info.get("web_search_billing_unit") == "per_query" else 1
def _record_freed_cost(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
original_cost: float,
freed_cost: float,
) -> float:
reduced_cost: Final = original_cost - freed_cost
breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost)
sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload
{
**breakdown,
"total_cost": reduced_cost,
"original_cost": breakdown.get("original_cost", original_cost),
"discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost,
}
)
sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload
kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"]
return reduced_cost
async def apply_search_free_tier(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
response_cost: float,
completion_response: object,
counter: SharedCounter | None,
now: datetime,
) -> float:
"""Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged."""
model_info: Final = sl_object["model_map_information"]["model_map_value"]
allowance: Final = model_info.get("search_free_requests") if model_info is not None else None
usage: Final = _usage_of(completion_response)
if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]:
return response_cost
units: Final = _billable_units(usage, model_info)
if units == 0 or kwargs.get(_COUNTED_KWARG):
return response_cost
if counter is None:
verbose_proxy_logger.warning(
"apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price"
)
return response_cost
key: Final = search_free_tier_key(allowance, now)
try:
total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now))
except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row
verbose_proxy_logger.warning(
"search free tier counter %s is unavailable, grounding is billed at full price", key
)
return response_cost
kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once
freed: Final = freed_units(allowance["count"], units, int(total_after))
if freed == 0:
return response_cost
freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units)
return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost)

7
litellm/types/utils.py
View file

@ -224,6 +224,12 @@ class SearchContextCostPerQuery(TypedDict, total=False):
search_context_size_high: float
class SearchFreeRequests(TypedDict):
count: ReadOnly[int]
period: ReadOnly[Literal["day", "month"]]
shared_group: ReadOnly[str]
class AgenticLoopParams(TypedDict, total=False):
"""
Parameters passed to agentic loop hooks (e.g., WebSearch interception).
@ -402,6 +408,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x)
search_free_requests: ReadOnly[SearchFreeRequests | None]
google_maps_grounding_cost_per_query: ReadOnly[float | None]
citation_cost_per_token: float | None # Cost per citation token for Perplexity
tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope

1
litellm/utils.py
View file

@ -6349,6 +6349,7 @@ def _get_model_info_helper(
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),

47
model_prices_and_context_window.json
View file

@ -26878,6 +26878,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -26990,6 +26991,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27041,6 +27043,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27087,6 +27090,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27129,6 +27133,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27219,6 +27224,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27280,6 +27286,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27339,6 +27346,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27425,6 +27433,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27479,6 +27488,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27525,6 +27535,7 @@
"supports_url_context": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27712,6 +27723,7 @@
"supports_video_input": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -27779,6 +27791,7 @@
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27834,6 +27847,7 @@
"supports_web_search": true,
"supports_url_context": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27890,6 +27904,7 @@
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -27940,6 +27955,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28001,6 +28017,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28064,6 +28081,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28123,6 +28141,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28182,6 +28201,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28328,6 +28348,7 @@
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28389,6 +28410,7 @@
"output_cost_per_token_above_200k_tokens_priority": 3.24e-05,
"cache_read_input_token_cost_priority": 3.6e-07,
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28716,6 +28738,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28760,6 +28783,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28821,6 +28845,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -28970,6 +28995,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 8000000,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29080,6 +29106,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29170,6 +29197,7 @@
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29301,6 +29329,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 250000,
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29504,6 +29533,7 @@
"supports_vision": true,
"supports_web_search": true,
"tpm": 800000,
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"search_context_cost_per_query": {
"search_context_size_low": 0.035,
"search_context_size_medium": 0.035,
@ -29601,6 +29631,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29661,6 +29692,7 @@
"supports_web_search": true,
"supports_native_streaming": true,
"tpm": 250000,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29717,6 +29749,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29780,6 +29813,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29845,6 +29879,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29906,6 +29941,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -29967,6 +30003,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30062,6 +30099,7 @@
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30125,6 +30163,7 @@
"cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07,
"cache_read_input_token_cost_batches": 2e-07,
"cache_read_input_token_cost_flex": 2e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30180,6 +30219,7 @@
"input_cost_per_audio_token_priority": 1.8e-06,
"output_cost_per_token_priority": 5.4e-06,
"cache_read_input_token_cost_priority": 9e-08,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30272,6 +30312,7 @@
"input_cost_per_audio_token_priority": 2.7e-06,
"output_cost_per_token_priority": 1.62e-05,
"cache_read_input_token_cost_priority": 2.7e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30335,6 +30376,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30394,6 +30436,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -30453,6 +30496,7 @@
"input_cost_per_token_priority": 1.35e-06,
"output_cost_per_token_priority": 6.75e-06,
"cache_read_input_token_cost_priority": 1.35e-07,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50523,6 +50567,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50585,6 +50630,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,
@ -50645,6 +50691,7 @@
"supports_vision": true,
"supports_web_search": true,
"supports_native_streaming": true,
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"search_context_cost_per_query": {
"search_context_size_low": 0.014,
"search_context_size_medium": 0.014,

27
model_prices_and_context_window.schema.json
View file

@ -916,6 +916,33 @@
},
"additionalProperties": false
},
"search_free_requests": {
"type": "object",
"description": "Free Google Search grounding requests per period, shared across models in the same shared_group.",
"required": [
"count",
"period",
"shared_group"
],
"additionalProperties": false,
"properties": {
"count": {
"type": "integer",
"minimum": 0
},
"period": {
"type": "string",
"enum": [
"day",
"month"
]
},
"shared_group": {
"type": "string",
"pattern": "\\S"
}
}
},
"source": {
"type": "string",
"description": "URL of the provider pricing/model page this entry was taken from."

5
tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY
from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth
@ -27,7 +28,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.mark.asyncio
@ -2936,7 +2937,6 @@ def test_autonomous_agent_cost_tracking_needs_no_human_or_virtual_key(agent_id:
assert _should_track_cost_callback(
user_api_key=None, user_id=None, team_id=None, end_user_id=None, call_type="acompletion", agent_id=agent_id
) is expected
||||||| parent of b217e5fa70 (fixup! feat(proxy): write one spend row per billed timed-out retry attempt (#47416))
class _SharedCounter:
@ -2958,7 +2958,6 @@ def _grounded_gemini_kwargs(response_cost: float) -> dict:
"response_cost": response_cost,
"request_tags": None,
"cache_hit": None,
"custom_llm_provider": "vertex_ai",
"cost_breakdown": {
"input_cost": 0.0,
"output_cost": 0.0,

194
tests/unit/proxy/hooks/test_search_free_tier.py Normal file
View file

@ -0,0 +1,194 @@
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.hooks.search_free_tier import (
apply_search_free_tier,
freed_units,
search_free_tier_key,
search_free_tier_ttl,
)
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc)
RATE: Final = 0.01
class _Counter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
self.ttls: dict[str, int | None] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
self.ttls[key] = ttl
return self.totals[key]
class _BrokenCounter:
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
raise ConnectionError("redis is down")
def _model_info(billing_unit: str, count: int, period: str = "month") -> dict:
return {
"litellm_provider": "vertex_ai",
"mode": "chat",
"web_search_billing_unit": billing_unit,
"search_context_cost_per_query": {"search_context_size_medium": RATE},
"search_free_requests": {"count": count, "period": period, "shared_group": "test-group"},
}
def _response(queries: int | None) -> ModelResponse:
details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None
return ModelResponse(
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details)
)
def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]:
sl_object = {
"response_cost": response_cost,
"cache_hit": cache_hit,
"cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost},
"model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info},
}
return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object
@pytest.mark.parametrize(
("allowance", "units", "total_after", "expected"),
[
(5000, 5, 5, 5),
(5000, 5, 5000, 5),
(5000, 10, 5005, 5),
(5000, 5, 5010, 0),
(2, 5, 5, 2),
(0, 3, 3, 0),
],
)
def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected):
assert freed_units(allowance, units, total_after) == expected
def test_counter_key_is_per_group_and_utc_period():
assert search_free_tier_key({"count": 1, "period": "month", "shared_group": "g3"}, NOW) == (
"litellm:search_free_tier:g3:2026-10"
)
assert search_free_tier_key({"count": 1, "period": "day", "shared_group": "g2"}, NOW) == (
"litellm:search_free_tier:g2:2026-10-20"
)
@pytest.mark.parametrize(
("period", "boundary"),
[("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))],
)
def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary):
expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW))
assert boundary < expiry < boundary + timedelta(hours=2)
@pytest.mark.asyncio
async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call():
counter = _Counter()
model_info = _model_info("per_query", count=7)
reduced = []
for _ in range(3):
kwargs, sl_object = _call(model_info, response_cost=1.0)
cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"]))
(c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced
assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE)
assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE)
assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5
assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE)
assert b2["discount_amount"] == pytest.approx(2 * RATE)
assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 15}
assert counter.ttls["litellm:search_free_tier:test-group:2026-10"] > 0
@pytest.mark.asyncio
async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count():
counter = _Counter()
model_info = _model_info("per_prompt", count=1, period="day")
kwargs, sl_object = _call(model_info, response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
kwargs, sl_object = _call(model_info, response_cost=1.0)
second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
assert first == pytest.approx(1.0 - RATE)
assert second == 1.0
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10-20": 2}
@pytest.mark.asyncio
async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
response = {
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3},
}
}
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", [None, _BrokenCounter()])
async def test_full_price_when_no_counter_is_available(counter):
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()])
async def test_requests_without_grounding_never_touch_the_counter(response):
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_models_without_an_allowance_never_touch_the_counter():
counter = _Counter()
model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"}
kwargs, sl_object = _call(model_info, response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_a_second_callback_run_on_the_same_request_counts_it_once():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW)
assert first == second == pytest.approx(1.0 - 5 * RATE)
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}
@pytest.mark.asyncio
async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True)
assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0
assert counter.totals == {}
assert "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
async def test_freed_cost_never_exceeds_the_billed_cost():
counter = _Counter()
billed = 3 * RATE
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed)
assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0
assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed)
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}

28
tests/unit/test_model_prices_schema.py
View file

@ -10,11 +10,13 @@ from typing import Final
import jsonschema
import pytest
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
from litellm.types.utils import SearchFreeRequests
REPO_ROOT = Path(__file__).parents[2]
GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py"
@ -579,3 +581,29 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
"unknown_root-new_family_models",
"vertex_ai-new_family_models",
]
def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
return MappingProxyType(
{key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row}
)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None:
rows = _search_free_requests_rows(path)
assert rows
allowances = {
key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items()
}
for key, row in rows.items():
assert row.get("supports_web_search") is True, key
assert row.get("search_context_cost_per_query"), key
assert allowances[key]["count"] > 0, key
for group in {allowance["shared_group"] for allowance in allowances.values()}:
same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group}
assert len(same_group) == 1, (group, same_group)
def test_search_free_requests_backup_rows_match_the_main_catalog() -> None:
assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH)

10
tests/unit/test_utils.py
View file

@ -1058,6 +1058,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "string",
"enum": ["per_prompt", "per_query"],
},
"search_free_requests": {
"type": "object",
"properties": {
"count": {"type": "integer", "minimum": 1},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string"},
},
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
},
"citation_cost_per_token": {"type": "number"},
"supported_modalities": {
"type": "array",