scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-27 06:13:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 97

feat(proxy): opt-in free-tier allowance for Google Search grounding spend (#47494)

Browse source 
* feat(proxy): opt-in free-tier allowance for Google Search grounding spend

* fix(proxy): declare the search free tier counter's service target

* test(cost_map): allow search_free_requests in the cost map schema

* fix(proxy): skip cache hits and never free more than the billed cost in the search free tier

---------

Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Kerry Lu 2026-10-20 10:50:26 -07:00 • committed by GitHub
parent b245cc0d26
commit e63aec029a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 564 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/__init__.py
View file

@ -551,6 +551,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
apply_search_free_tier: bool = False
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:

1
litellm/constants.py
View file

@ -1979,6 +1979,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [
"public_model_groups",
"public_model_groups_links",
"cost_discount_config",
"apply_search_free_tier",
"cost_margin_config",
"block_requests_for_models_without_pricing",
"budget_exceeded_throttle_percentage",

47
litellm/model_prices_and_context_window_backup.json
View file

@ -28948,6 +28948,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_token_batches": 1.5e-07,
@ -29062,6 +29063,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
@ -29114,6 +29116,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image": {
@ -29164,6 +29167,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
@ -29207,6 +29211,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
@ -29302,6 +29307,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29363,6 +29369,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -29422,6 +29429,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29508,6 +29516,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-08,
"input_cost_per_token_batches": 5e-08,
@ -29562,6 +29571,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@ -29608,6 +29618,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@ -29795,6 +29806,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"input_cost_per_token_batches": 6.25e-07,
@ -29832,6 +29844,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -29912,6 +29925,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29968,6 +29982,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3-flash-preview": {
@ -29992,6 +30007,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30074,6 +30090,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -30138,6 +30155,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30198,6 +30216,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30224,6 +30243,7 @@
"prompt_cache_min_tokens": 4096.0,
"regional_endpoint_uplift_multiplier": 1.1,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30369,6 +30389,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30455,6 +30476,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30782,6 +30804,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
@ -30826,6 +30849,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-preview": {
@ -30887,6 +30911,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -31036,6 +31061,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_audio_token_priority": 1.8e-06,
@ -31148,6 +31174,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": true
},
@ -31212,6 +31239,7 @@
"output_cost_per_token_batches": 1.5e-06,
"rpm": 1000.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -31362,6 +31390,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 1.5e-07,
"input_cost_per_audio_token_priority": 5.4e-07,
@ -31567,6 +31596,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_batches": 6.25e-07,
"input_cost_per_token_flex": 6.25e-07,
@ -31664,6 +31694,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
@ -31725,6 +31756,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -31781,6 +31813,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 5e-07,
@ -31844,6 +31877,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -31909,6 +31943,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -31970,6 +32005,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32031,6 +32067,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32126,6 +32163,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@ -32189,6 +32227,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@ -32217,6 +32256,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -32331,6 +32371,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -32395,6 +32436,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32455,6 +32497,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32480,6 +32523,7 @@
"output_cost_per_token_priority": 6.75e-06,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -52449,6 +52493,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -52511,6 +52556,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -52571,6 +52617,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},

18
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -1,7 +1,7 @@
import asyncio
import traceback
from collections.abc import Callable, Mapping, Sequence
from datetime import datetime
from datetime import datetime, timezone
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol, cast
@ -33,6 +33,7 @@ from litellm.proxy.db.db_spend_update_writer import (
debitable_model_access_groups,
get_llm_router,
)
from litellm.proxy.hooks.search_free_tier import apply_search_free_tier
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route
from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope
from litellm.proxy.spend_tracking.spend_event import (
@ -487,8 +488,21 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
raw_cost: Final = _str_keyed_fields(
sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
).get("response_cost")
priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
await apply_search_free_tier(
kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload
sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists
response_cost=priced_cost,
completion_response=completion_response,
counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache,
now=datetime.now(timezone.utc),
)
if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None
else priced_cost
)
tags: Final = _get_request_tags_for_cost_tracking(
sl_object=sl_object,

123
litellm/proxy/hooks/search_free_tier.py Normal file
View file

@ -0,0 +1,123 @@
from collections.abc import MutableMapping
from datetime import datetime
from types import MappingProxyType
from typing import Final, Protocol
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from litellm._internal_context import with_service_target
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time
from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage
from litellm.llms.gemini.cost_calculator import cost_per_web_search_request
from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage
_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"})
_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"})
_TTL_GRACE_SECONDS: Final = 3600
_COUNTED_KWARG: Final = "search_free_tier_counted"
_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown)
class SharedCounter(Protocol):
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ...
class _UsageCarrier(BaseModel):
model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
usage: Usage | None = None
def freed_units(allowance: int, units: int, total_after: int) -> int:
"""Units of this request that landed under the allowance, given the shared total after they were added."""
return max(0, min(units, allowance - (total_after - units)))
def search_free_tier_key(allowance: SearchFreeRequests, now: datetime) -> str:
return f"litellm:search_free_tier:{allowance['shared_group']}:{now.strftime(_PERIOD_STAMP[allowance['period']])}"
def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int:
reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC")
return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS
def _usage_of(completion_response: object) -> Usage | None:
try:
return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage
except ValidationError:
return None
def _grounding_queries(usage: Usage) -> int:
details: Final = usage.prompt_tokens_details
from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None
return from_details or get_web_search_requests_from_usage(usage) or 0
def _billable_units(usage: Usage, model_info: ModelInfo) -> int:
queries: Final = _grounding_queries(usage)
if queries == 0:
return 0
return queries if model_info.get("web_search_billing_unit") == "per_query" else 1
def _record_freed_cost(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
original_cost: float,
freed_cost: float,
) -> float:
reduced_cost: Final = original_cost - freed_cost
breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost)
sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload
{
**breakdown,
"total_cost": reduced_cost,
"original_cost": breakdown.get("original_cost", original_cost),
"discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost,
}
)
sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload
kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"]
return reduced_cost
@with_service_target("search_free_tier")
async def apply_search_free_tier(
kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it
sl_object: StandardLoggingPayload,
response_cost: float,
completion_response: object,
counter: SharedCounter | None,
now: datetime,
) -> float:
"""Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged."""
model_info: Final = sl_object["model_map_information"]["model_map_value"]
allowance: Final = model_info.get("search_free_requests") if model_info is not None else None
usage: Final = _usage_of(completion_response)
if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]:
return response_cost
units: Final = _billable_units(usage, model_info)
if units == 0 or kwargs.get(_COUNTED_KWARG):
return response_cost
if counter is None:
verbose_proxy_logger.warning(
"apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price"
)
return response_cost
key: Final = search_free_tier_key(allowance, now)
try:
total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now))
except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row
verbose_proxy_logger.warning(
"search free tier counter %s is unavailable, grounding is billed at full price", key
)
return response_cost
kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once
freed: Final = freed_units(allowance["count"], units, int(total_after))
if freed == 0:
return response_cost
freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units)
return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost)

7
litellm/types/utils.py
View file

@ -245,6 +245,12 @@ class SearchContextCostPerQuery(TypedDict, total=False):
search_context_size_high: float
class SearchFreeRequests(TypedDict):
count: ReadOnly[int]
period: ReadOnly[Literal["day", "month"]]
shared_group: ReadOnly[str]
class AgenticLoopParams(TypedDict, total=False):
"""
Parameters passed to agentic loop hooks (e.g., WebSearch interception).
@ -436,6 +442,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
web_search_billing_unit: (
Literal["per_query", "per_prompt"] | None
) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x)
search_free_requests: ReadOnly[SearchFreeRequests | None]
google_maps_grounding_cost_per_query: ReadOnly[float | None]
citation_cost_per_token: float | None # Cost per citation token for Perplexity
tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope

1
litellm/utils.py
View file

@ -6878,6 +6878,7 @@ def get_model_info_helper(
supports_computer_use=_model_info.get("supports_computer_use", None),
search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None),
web_search_billing_unit=_model_info.get("web_search_billing_unit", None),
search_free_requests=_model_info.get("search_free_requests", None),
google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None),
tpm=_model_info.get("tpm", None),
rpm=_model_info.get("rpm", None),

47
model_prices_and_context_window.json
View file

@ -28948,6 +28948,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_token_batches": 1.5e-07,
@ -29062,6 +29063,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
@ -29114,6 +29116,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image": {
@ -29164,6 +29167,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
@ -29207,6 +29211,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-lite-image": {
@ -29302,6 +29307,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29363,6 +29369,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -29422,6 +29429,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29508,6 +29516,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-08,
"input_cost_per_token_batches": 5e-08,
@ -29562,6 +29571,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@ -29608,6 +29618,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"supports_image_size": false
},
@ -29795,6 +29806,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_above_200k_tokens_priority": 4.5e-06,
"input_cost_per_token_batches": 6.25e-07,
@ -29832,6 +29844,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -29912,6 +29925,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -29968,6 +29982,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"vertex_ai/gemini-3-flash-preview": {
@ -29992,6 +30007,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30074,6 +30090,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -30138,6 +30155,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30198,6 +30216,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30224,6 +30243,7 @@
"prompt_cache_min_tokens": 4096.0,
"regional_endpoint_uplift_multiplier": 1.1,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30369,6 +30389,7 @@
"output_cost_per_token_priority": 2.16e-05,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -30455,6 +30476,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -30782,6 +30804,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": false
},
@ -30826,6 +30849,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query"
},
"gemini/gemini-3.1-flash-lite-preview": {
@ -30887,6 +30911,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -31036,6 +31061,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 5e-07,
"input_cost_per_audio_token_priority": 1.8e-06,
@ -31148,6 +31174,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"supports_reasoning": true
},
@ -31212,6 +31239,7 @@
"output_cost_per_token_batches": 1.5e-06,
"rpm": 1000.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://ai.google.dev/gemini-api/docs/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -31362,6 +31390,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_audio_token_batches": 1.5e-07,
"input_cost_per_audio_token_priority": 5.4e-07,
@ -31567,6 +31596,7 @@
"search_context_size_medium": 0.035,
"search_context_size_high": 0.035
},
"search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"},
"google_maps_grounding_cost_per_query": 0.025,
"input_cost_per_token_batches": 6.25e-07,
"input_cost_per_token_flex": 6.25e-07,
@ -31664,6 +31694,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
@ -31725,6 +31756,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -31781,6 +31813,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 5e-07,
@ -31844,6 +31877,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -31909,6 +31943,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -31970,6 +32005,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32031,6 +32067,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32126,6 +32163,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@ -32189,6 +32227,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_flex": 1e-06,
@ -32217,6 +32256,7 @@
"output_cost_per_token_flex": 1.5e-06,
"output_cost_per_token_priority": 5.4e-06,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -32331,6 +32371,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_token_batches": 7.5e-07,
@ -32395,6 +32436,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32455,6 +32497,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -32480,6 +32523,7 @@
"output_cost_per_token_priority": 6.75e-06,
"prompt_cache_min_tokens": 4096.0,
"search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"supported_endpoints": [
"/v1/chat/completions",
@ -52449,6 +52493,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},
@ -52511,6 +52556,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07
@ -52571,6 +52617,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014
},

90
tests/unit/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -29,7 +29,7 @@ from litellm.proxy.utils import ProxyUpdateSpend
from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event
from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage
from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage
@pytest.mark.asyncio
@ -3182,3 +3182,91 @@ async def test_track_cost_callback_enqueue_emits_no_service_span(): # test-qual
assert update_database.await_count == 1, "the spend enqueue itself must still run"
assert success_hook.await_count == 0, [call.kwargs for call in success_hook.await_args_list]
class _SharedCounter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
return self.totals[key]
def _grounded_gemini_kwargs(response_cost: float) -> dict:
return {
"model": "vertex_ai/gemini-test",
"call_type": "acompletion",
"response_cost": response_cost,
"litellm_params": {"metadata": {"user_api_key": "sk-test", "user_api_key_user_id": "user-1"}},
"standard_logging_object": {
"response_cost": response_cost,
"request_tags": None,
"cache_hit": None,
"cost_breakdown": {
"input_cost": 0.0,
"output_cost": 0.0,
"tool_usage_cost": 0.5,
"total_cost": response_cost,
},
"model_map_information": {
"model_map_key": "vertex_ai/gemini-test",
"model_map_value": {
"litellm_provider": "vertex_ai",
"mode": "chat",
"web_search_billing_unit": "per_query",
"search_context_cost_per_query": {"search_context_size_medium": 0.1},
"search_free_requests": {"count": 100, "period": "month", "shared_group": "test"},
},
},
},
"stream": False,
}
@pytest.mark.asyncio
@pytest.mark.parametrize("opted_in", [True, False])
async def test_track_cost_callback_frees_grounding_under_the_allowance_only_when_opted_in(monkeypatch, opted_in):
monkeypatch.setattr(litellm, "apply_search_free_tier", opted_in, raising=False)
counter = _SharedCounter()
kwargs = _grounded_gemini_kwargs(response_cost=1.0)
response = ModelResponse(
usage=Usage(
prompt_tokens=10,
completion_tokens=5,
total_tokens=15,
prompt_tokens_details=PromptTokensDetailsWrapper(web_search_requests=5),
)
)
with (
patch( # test-quality-ok: increment_spend_counters is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock
),
patch( # test-quality-ok: update_cache is a proxy_server global the callback reads lazily, no seam
"litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock
),
patch( # test-quality-ok: callback imports proxy_logging_obj off proxy_server in its body, no seam
"litellm.proxy.proxy_server.proxy_logging_obj"
) as mock_proxy_logging,
):
mock_proxy_logging.failed_tracking_alert = AsyncMock()
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock(return_value=True)
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
mock_proxy_logging.internal_usage_cache.dual_cache.redis_cache = counter
await ProxyDBLogger()._PROXY_track_cost_callback(
kwargs=kwargs, completion_response=response, start_time=datetime.now(), end_time=datetime.now()
)
await asyncio.sleep(0)
mock_proxy_logging.failed_tracking_alert.assert_not_called()
mock_proxy_logging.db_spend_update_writer.update_database.assert_awaited_once()
breakdown = kwargs["standard_logging_object"]["cost_breakdown"]
if opted_in:
assert kwargs["response_cost"] == pytest.approx(1.0 - 5 * 0.1)
assert breakdown["original_cost"] == 1.0 and breakdown["discount_amount"] == pytest.approx(5 * 0.1)
assert sum(counter.totals.values()) == 5
else:
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in breakdown
assert counter.totals == {}

194
tests/unit/proxy/hooks/test_search_free_tier.py Normal file
View file

@ -0,0 +1,194 @@
from datetime import datetime, timedelta, timezone
from typing import Final
import pytest
from litellm.proxy.hooks.search_free_tier import (
apply_search_free_tier,
freed_units,
search_free_tier_key,
search_free_tier_ttl,
)
from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage
NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc)
RATE: Final = 0.01
class _Counter:
def __init__(self) -> None:
self.totals: dict[str, float] = {}
self.ttls: dict[str, int | None] = {}
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
self.totals[key] = self.totals.get(key, 0) + value
self.ttls[key] = ttl
return self.totals[key]
class _BrokenCounter:
async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float:
raise ConnectionError("redis is down")
def _model_info(billing_unit: str, count: int, period: str = "month") -> dict:
return {
"litellm_provider": "vertex_ai",
"mode": "chat",
"web_search_billing_unit": billing_unit,
"search_context_cost_per_query": {"search_context_size_medium": RATE},
"search_free_requests": {"count": count, "period": period, "shared_group": "test-group"},
}
def _response(queries: int | None) -> ModelResponse:
details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None
return ModelResponse(
usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details)
)
def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]:
sl_object = {
"response_cost": response_cost,
"cache_hit": cache_hit,
"cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost},
"model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info},
}
return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object
@pytest.mark.parametrize(
("allowance", "units", "total_after", "expected"),
[
(5000, 5, 5, 5),
(5000, 5, 5000, 5),
(5000, 10, 5005, 5),
(5000, 5, 5010, 0),
(2, 5, 5, 2),
(0, 3, 3, 0),
],
)
def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected):
assert freed_units(allowance, units, total_after) == expected
def test_counter_key_is_per_group_and_utc_period():
assert search_free_tier_key({"count": 1, "period": "month", "shared_group": "g3"}, NOW) == (
"litellm:search_free_tier:g3:2026-10"
)
assert search_free_tier_key({"count": 1, "period": "day", "shared_group": "g2"}, NOW) == (
"litellm:search_free_tier:g2:2026-10-20"
)
@pytest.mark.parametrize(
("period", "boundary"),
[("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))],
)
def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary):
expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW))
assert boundary < expiry < boundary + timedelta(hours=2)
@pytest.mark.asyncio
async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call():
counter = _Counter()
model_info = _model_info("per_query", count=7)
reduced = []
for _ in range(3):
kwargs, sl_object = _call(model_info, response_cost=1.0)
cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"]))
(c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced
assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE)
assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE)
assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5
assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE)
assert b2["discount_amount"] == pytest.approx(2 * RATE)
assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 15}
assert counter.ttls["litellm:search_free_tier:test-group:2026-10"] > 0
@pytest.mark.asyncio
async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count():
counter = _Counter()
model_info = _model_info("per_prompt", count=1, period="day")
kwargs, sl_object = _call(model_info, response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
kwargs, sl_object = _call(model_info, response_cost=1.0)
second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
assert first == pytest.approx(1.0 - RATE)
assert second == 1.0
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10-20": 2}
@pytest.mark.asyncio
async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
response = {
"usage": {
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
"prompt_tokens_details": {"web_search_requests": 3},
}
}
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE)
@pytest.mark.asyncio
@pytest.mark.parametrize("counter", [None, _BrokenCounter()])
async def test_full_price_when_no_counter_is_available(counter):
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()])
async def test_requests_without_grounding_never_touch_the_counter(response):
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_models_without_an_allowance_never_touch_the_counter():
counter = _Counter()
model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"}
kwargs, sl_object = _call(model_info, response_cost=1.0)
assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0
assert counter.totals == {}
@pytest.mark.asyncio
async def test_a_second_callback_run_on_the_same_request_counts_it_once():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0)
first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW)
second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW)
assert first == second == pytest.approx(1.0 - 5 * RATE)
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}
@pytest.mark.asyncio
async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero():
counter = _Counter()
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True)
assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0
assert counter.totals == {}
assert "discount_amount" not in sl_object["cost_breakdown"]
@pytest.mark.asyncio
async def test_freed_cost_never_exceeds_the_billed_cost():
counter = _Counter()
billed = 3 * RATE
kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed)
assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0
assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed)
assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5}

28
tests/unit/test_model_prices_and_context_window.py
View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.fallback_generalizations import match_capability
from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
from litellm.types.utils import SearchFreeRequests
REPO_ROOT = Path(__file__).parents[2]
PRICES_PATH = REPO_ROOT / "model_prices_and_context_window.json"
@ -721,6 +722,33 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered():
]
def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]:
prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes())
return MappingProxyType(
{key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row}
)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None:
rows = _search_free_requests_rows(path)
assert rows
allowances = {
key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items()
}
for key, row in rows.items():
assert row.get("supports_web_search") is True, key
assert row.get("search_context_cost_per_query"), key
assert allowances[key]["count"] > 0, key
for group in {allowance["shared_group"] for allowance in allowances.values()}:
same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group}
assert len(same_group) == 1, (group, same_group)
def test_search_free_requests_backup_rows_match_the_main_catalog() -> None:
assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH)
@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup"))
def test_volcengine_doubao_seed_2_1_pro_260915_row_matches_published_metadata(path: Path):
"""Volcengine Ark lists doubao-seed-2-1-pro-260915 at 6 / 30 CNY per million input / output tokens with

10
tests/unit/test_utils.py
View file

@ -1260,6 +1260,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
"type": "string",
"enum": ["per_prompt", "per_query"],
},
"search_free_requests": {
"type": "object",
"properties": {
"count": {"type": "integer", "minimum": 1},
"period": {"type": "string", "enum": ["day", "month"]},
"shared_group": {"type": "string"},
},
"required": ["count", "period", "shared_group"],
"additionalProperties": False,
},
"citation_cost_per_token": {"type": "number"},
"supported_modalities": {
"type": "array",