diff --git a/ci_cd/generate_model_prices_schema.py b/ci_cd/generate_model_prices_schema.py index a5a4b7f6f3b..8f7c9a343a3 100644 --- a/ci_cd/generate_model_prices_schema.py +++ b/ci_cd/generate_model_prices_schema.py @@ -91,6 +91,17 @@ OBJECT_KEYS: dict[str, JsonSchema] = { }, "additionalProperties": False, }, + "search_free_requests": { + "type": "object", + "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + "properties": { + "count": NONNEG_INTEGER, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": NON_BLANK_STRING, + }, + }, "guardrail_cost_per_unit": { "type": "object", "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).", diff --git a/litellm/__init__.py b/litellm/__init__.py index 81fd6ca6ca5..0999c5e2224 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -562,6 +562,7 @@ from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map, ma model_cost = get_model_cost_map(url=model_cost_map_url) cost_discount_config: Dict[str, float] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount +apply_search_free_tier: bool = False cost_margin_config: Dict[ str, Union[float, Dict[str, float]] ] = {} # Provider-specific or global cost margins. Examples: diff --git a/litellm/constants.py b/litellm/constants.py index 9ee73ec7b98..1130754f320 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1896,6 +1896,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES: Final = [ "public_model_groups", "public_model_groups_links", "cost_discount_config", + "apply_search_free_tier", "cost_margin_config", "block_requests_for_models_without_pricing", "budget_exceeded_throttle_percentage", diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 78f17ee3f07..81bc29ac266 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -26878,6 +26878,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -26990,6 +26991,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27041,6 +27043,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27087,6 +27090,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27129,6 +27133,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27219,6 +27224,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27280,6 +27286,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27339,6 +27346,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27425,6 +27433,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27479,6 +27488,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27525,6 +27535,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27712,6 +27723,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27779,6 +27791,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27834,6 +27847,7 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27890,6 +27904,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27940,6 +27955,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28001,6 +28017,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28064,6 +28081,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28123,6 +28141,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28182,6 +28201,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28328,6 +28348,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28389,6 +28410,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28716,6 +28738,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28760,6 +28783,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28821,6 +28845,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28970,6 +28995,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29080,6 +29106,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29170,6 +29197,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29301,6 +29329,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29504,6 +29533,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 800000, + "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29601,6 +29631,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29661,6 +29692,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29717,6 +29749,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29780,6 +29813,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29845,6 +29879,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29906,6 +29941,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29967,6 +30003,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30062,6 +30099,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30125,6 +30163,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30180,6 +30219,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30272,6 +30312,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30335,6 +30376,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30394,6 +30436,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30453,6 +30496,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50523,6 +50567,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50585,6 +50630,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50645,6 +50691,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index ca4aa00e601..63a47dad696 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -1,9 +1,11 @@ import asyncio import traceback from collections.abc import Callable, Mapping, Sequence -from datetime import datetime +from datetime import datetime, timezone from typing import TYPE_CHECKING, Any, Final, Protocol, cast +from typing_extensions import TypeIs + import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.batch_utils import batch_cost_is_final @@ -30,6 +32,7 @@ from litellm.proxy.db.db_spend_update_writer import ( debitable_model_access_groups, get_llm_router, ) +from litellm.proxy.hooks.search_free_tier import apply_search_free_tier from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, metadata_variable_name_for_route from litellm.proxy.spend_tracking.spend_counter_batch import post_call_counter_keys, spend_counter_batch_scope from litellm.proxy.spend_tracking.spend_event import ( @@ -106,6 +109,16 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter: return proxy_logging_obj.db_spend_update_writer +def _is_str_keyed_mapping( + value: object, +) -> TypeIs[Mapping[str, object]]: # guard-ok: success-callback payloads are str-keyed dicts + return isinstance(value, Mapping) + + +def _str_keyed_fields(value: object) -> Mapping[str, object]: + return value if _is_str_keyed_mapping(value) else {} + + def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]: """The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any.""" attempts: Final = request_data.get("billed_failed_attempts") @@ -434,8 +447,21 @@ class _ProxyDBLogger(CustomLogger): key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None)) end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None) sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None) + raw_cost: Final = _str_keyed_fields( + sl_object if sl_object is not None else kwargs # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload + ).get("response_cost") + priced_cost: Final = raw_cost if isinstance(raw_cost, (int, float)) else None response_cost: Final = ( - sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None) + await apply_search_free_tier( + kwargs=kwargs, # pyright: ignore[reportUnknownArgumentType] # untyped success-callback payload + sl_object=sl_object, # pyright: ignore[reportUnknownArgumentType] # payload carries untyped lists + response_cost=priced_cost, + completion_response=completion_response, + counter=proxy_logging_obj.internal_usage_cache.dual_cache.redis_cache, + now=datetime.now(timezone.utc), + ) + if litellm.apply_search_free_tier and sl_object is not None and priced_cost is not None + else priced_cost ) tags: Final = _get_request_tags_for_cost_tracking( sl_object=sl_object, diff --git a/litellm/proxy/hooks/search_free_tier.py b/litellm/proxy/hooks/search_free_tier.py new file mode 100644 index 00000000000..b9beb8599b3 --- /dev/null +++ b/litellm/proxy/hooks/search_free_tier.py @@ -0,0 +1,121 @@ +from collections.abc import MutableMapping +from datetime import datetime +from types import MappingProxyType +from typing import Final, Protocol + +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm._logging import verbose_proxy_logger +from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time +from litellm.litellm_core_utils.llm_cost_calc.utils import get_web_search_requests_from_usage +from litellm.llms.gemini.cost_calculator import cost_per_web_search_request +from litellm.types.utils import CostBreakdown, ModelInfo, SearchFreeRequests, StandardLoggingPayload, Usage + +_PERIOD_DURATION: Final = MappingProxyType({"day": "1d", "month": "1mo"}) +_PERIOD_STAMP: Final = MappingProxyType({"day": "%Y-%m-%d", "month": "%Y-%m"}) +_TTL_GRACE_SECONDS: Final = 3600 +_COUNTED_KWARG: Final = "search_free_tier_counted" +_COST_BREAKDOWN: Final = TypeAdapter(CostBreakdown) + + +class SharedCounter(Protocol): + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: ... + + +class _UsageCarrier(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True) + + usage: Usage | None = None + + +def freed_units(allowance: int, units: int, total_after: int) -> int: + """Units of this request that landed under the allowance, given the shared total after they were added.""" + return max(0, min(units, allowance - (total_after - units))) + + +def search_free_tier_key(allowance: SearchFreeRequests, now: datetime) -> str: + return f"litellm:search_free_tier:{allowance['shared_group']}:{now.strftime(_PERIOD_STAMP[allowance['period']])}" + + +def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int: + reset: Final = get_next_standardized_reset_time(_PERIOD_DURATION[allowance["period"]], now, "UTC") + return int((reset - now).total_seconds()) + _TTL_GRACE_SECONDS + + +def _usage_of(completion_response: object) -> Usage | None: + try: + return _UsageCarrier.model_validate(completion_response, from_attributes=True).usage + except ValidationError: + return None + + +def _grounding_queries(usage: Usage) -> int: + details: Final = usage.prompt_tokens_details + from_details: Final = getattr(details, "web_search_requests", None) if details is not None else None + return from_details or get_web_search_requests_from_usage(usage) or 0 + + +def _billable_units(usage: Usage, model_info: ModelInfo) -> int: + queries: Final = _grounding_queries(usage) + if queries == 0: + return 0 + return queries if model_info.get("web_search_billing_unit") == "per_query" else 1 + + +def _record_freed_cost( + kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it + sl_object: StandardLoggingPayload, + original_cost: float, + freed_cost: float, +) -> float: + reduced_cost: Final = original_cost - freed_cost + breakdown: Final = sl_object["cost_breakdown"] or CostBreakdown(total_cost=original_cost) + sl_object["cost_breakdown"] = _COST_BREAKDOWN.validate_python( # rebind-ok: the spend row reads this payload + { + **breakdown, + "total_cost": reduced_cost, + "original_cost": breakdown.get("original_cost", original_cost), + "discount_amount": breakdown.get("discount_amount", 0.0) + freed_cost, + } + ) + sl_object["response_cost"] = reduced_cost # rebind-ok: the spend row reads this payload + kwargs["response_cost"] = reduced_cost # rebind-ok: the spend row reads kwargs["response_cost"] + return reduced_cost + + +async def apply_search_free_tier( + kwargs: MutableMapping[str, object], # mutable-ok: writes the reduced cost where the spend row reads it + sl_object: StandardLoggingPayload, + response_cost: float, + completion_response: object, + counter: SharedCounter | None, + now: datetime, +) -> float: + """Zero the grounding queries that landed under the model's shared free allowance, everything else is unchanged.""" + model_info: Final = sl_object["model_map_information"]["model_map_value"] + allowance: Final = model_info.get("search_free_requests") if model_info is not None else None + usage: Final = _usage_of(completion_response) + if model_info is None or allowance is None or usage is None or sl_object["cache_hit"]: + return response_cost + units: Final = _billable_units(usage, model_info) + if units == 0 or kwargs.get(_COUNTED_KWARG): + return response_cost + if counter is None: + verbose_proxy_logger.warning( + "apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price" + ) + return response_cost + key: Final = search_free_tier_key(allowance, now) + try: + total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now)) + except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row + verbose_proxy_logger.warning( + "search free tier counter %s is unavailable, grounding is billed at full price", key + ) + return response_cost + kwargs[_COUNTED_KWARG] = True # rebind-ok: the proxy runs this callback twice per request, count it once + freed: Final = freed_units(allowance["count"], units, int(total_after)) + if freed == 0: + return response_cost + freed_cost: Final = min(response_cost, cost_per_web_search_request(usage, model_info) * freed / units) + return _record_freed_cost(kwargs, sl_object, response_cost, freed_cost) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 30de80a460d..96ea31fa33c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -224,6 +224,12 @@ class SearchContextCostPerQuery(TypedDict, total=False): search_context_size_high: float +class SearchFreeRequests(TypedDict): + count: ReadOnly[int] + period: ReadOnly[Literal["day", "month"]] + shared_group: ReadOnly[str] + + class AgenticLoopParams(TypedDict, total=False): """ Parameters passed to agentic loop hooks (e.g., WebSearch interception). @@ -402,6 +408,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): web_search_billing_unit: ( Literal["per_query", "per_prompt"] | None ) # "per_query" (Gemini 3.x) or "per_prompt" (Gemini 2.x) + search_free_requests: ReadOnly[SearchFreeRequests | None] google_maps_grounding_cost_per_query: ReadOnly[float | None] citation_cost_per_token: float | None # Cost per citation token for Perplexity tiered_pricing: list[dict[str, Any]] | None # Tiered pricing structure for models like Dashscope diff --git a/litellm/utils.py b/litellm/utils.py index 5648db2c3b7..a6d3ddac5a1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6349,6 +6349,7 @@ def _get_model_info_helper( supports_computer_use=_model_info.get("supports_computer_use", None), search_context_cost_per_query=_model_info.get("search_context_cost_per_query", None), web_search_billing_unit=_model_info.get("web_search_billing_unit", None), + search_free_requests=_model_info.get("search_free_requests", None), google_maps_grounding_cost_per_query=_model_info.get("google_maps_grounding_cost_per_query", None), tpm=_model_info.get("tpm", None), rpm=_model_info.get("rpm", None), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 78f17ee3f07..81bc29ac266 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -26878,6 +26878,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -26990,6 +26991,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27041,6 +27043,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27087,6 +27090,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27129,6 +27133,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27219,6 +27224,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27280,6 +27286,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27339,6 +27346,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27425,6 +27433,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27479,6 +27488,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27525,6 +27535,7 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27712,6 +27723,7 @@ "supports_video_input": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -27779,6 +27791,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27834,6 +27847,7 @@ "supports_web_search": true, "supports_url_context": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27890,6 +27904,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -27940,6 +27955,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28001,6 +28017,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28064,6 +28081,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28123,6 +28141,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28182,6 +28201,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28328,6 +28348,7 @@ "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28389,6 +28410,7 @@ "output_cost_per_token_above_200k_tokens_priority": 3.24e-05, "cache_read_input_token_cost_priority": 3.6e-07, "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28716,6 +28738,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28760,6 +28783,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28821,6 +28845,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -28970,6 +28995,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 8000000, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29080,6 +29106,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29170,6 +29197,7 @@ "supports_system_messages": true, "supports_vision": true, "supports_web_search": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29301,6 +29329,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 250000, + "search_free_requests": {"count": 1500, "period": "day", "shared_group": "gemini-2-flash"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29504,6 +29533,7 @@ "supports_vision": true, "supports_web_search": true, "tpm": 800000, + "search_free_requests": {"count": 10000, "period": "day", "shared_group": "gemini-2.5-pro"}, "search_context_cost_per_query": { "search_context_size_low": 0.035, "search_context_size_medium": 0.035, @@ -29601,6 +29631,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29661,6 +29692,7 @@ "supports_web_search": true, "supports_native_streaming": true, "tpm": 250000, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29717,6 +29749,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29780,6 +29813,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29845,6 +29879,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29906,6 +29941,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -29967,6 +30003,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30062,6 +30099,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30125,6 +30163,7 @@ "cache_read_input_token_cost_above_200k_tokens_priority": 7.2e-07, "cache_read_input_token_cost_batches": 2e-07, "cache_read_input_token_cost_flex": 2e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30180,6 +30219,7 @@ "input_cost_per_audio_token_priority": 1.8e-06, "output_cost_per_token_priority": 5.4e-06, "cache_read_input_token_cost_priority": 9e-08, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30272,6 +30312,7 @@ "input_cost_per_audio_token_priority": 2.7e-06, "output_cost_per_token_priority": 1.62e-05, "cache_read_input_token_cost_priority": 2.7e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30335,6 +30376,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30394,6 +30436,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -30453,6 +30496,7 @@ "input_cost_per_token_priority": 1.35e-06, "output_cost_per_token_priority": 6.75e-06, "cache_read_input_token_cost_priority": 1.35e-07, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50523,6 +50567,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50585,6 +50630,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, @@ -50645,6 +50691,7 @@ "supports_vision": true, "supports_web_search": true, "supports_native_streaming": true, + "search_free_requests": {"count": 5000, "period": "month", "shared_group": "gemini-3"}, "search_context_cost_per_query": { "search_context_size_low": 0.014, "search_context_size_medium": 0.014, diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index a59b2bd342d..1683e020b3b 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -916,6 +916,33 @@ }, "additionalProperties": false }, + "search_free_requests": { + "type": "object", + "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", + "required": [ + "count", + "period", + "shared_group" + ], + "additionalProperties": false, + "properties": { + "count": { + "type": "integer", + "minimum": 0 + }, + "period": { + "type": "string", + "enum": [ + "day", + "month" + ] + }, + "shared_group": { + "type": "string", + "pattern": "\\S" + } + } + }, "source": { "type": "string", "description": "URL of the provider pricing/model page this entry was taken from." diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 2add2636615..d493754866b 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import litellm from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.internal_call_metadata import MODEL_ACCESS_GROUP_METADATA_KEY from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth @@ -27,7 +28,7 @@ from litellm.proxy.utils import ProxyUpdateSpend from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload -from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, Usage +from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage @pytest.mark.asyncio @@ -2936,7 +2937,6 @@ def test_autonomous_agent_cost_tracking_needs_no_human_or_virtual_key(agent_id: assert _should_track_cost_callback( user_api_key=None, user_id=None, team_id=None, end_user_id=None, call_type="acompletion", agent_id=agent_id ) is expected -||||||| parent of b217e5fa70 (fixup! feat(proxy): write one spend row per billed timed-out retry attempt (#47416)) class _SharedCounter: @@ -2958,7 +2958,6 @@ def _grounded_gemini_kwargs(response_cost: float) -> dict: "response_cost": response_cost, "request_tags": None, "cache_hit": None, - "custom_llm_provider": "vertex_ai", "cost_breakdown": { "input_cost": 0.0, "output_cost": 0.0, diff --git a/tests/unit/proxy/hooks/test_search_free_tier.py b/tests/unit/proxy/hooks/test_search_free_tier.py new file mode 100644 index 00000000000..3dc89c0dc72 --- /dev/null +++ b/tests/unit/proxy/hooks/test_search_free_tier.py @@ -0,0 +1,194 @@ +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest + +from litellm.proxy.hooks.search_free_tier import ( + apply_search_free_tier, + freed_units, + search_free_tier_key, + search_free_tier_ttl, +) +from litellm.types.utils import ModelResponse, PromptTokensDetailsWrapper, Usage + +NOW: Final = datetime(2026, 10, 20, 12, 30, tzinfo=timezone.utc) +RATE: Final = 0.01 + + +class _Counter: + def __init__(self) -> None: + self.totals: dict[str, float] = {} + self.ttls: dict[str, int | None] = {} + + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + self.totals[key] = self.totals.get(key, 0) + value + self.ttls[key] = ttl + return self.totals[key] + + +class _BrokenCounter: + async def async_increment(self, key: str, value: float, ttl: int | None = None) -> float: + raise ConnectionError("redis is down") + + +def _model_info(billing_unit: str, count: int, period: str = "month") -> dict: + return { + "litellm_provider": "vertex_ai", + "mode": "chat", + "web_search_billing_unit": billing_unit, + "search_context_cost_per_query": {"search_context_size_medium": RATE}, + "search_free_requests": {"count": count, "period": period, "shared_group": "test-group"}, + } + + +def _response(queries: int | None) -> ModelResponse: + details = PromptTokensDetailsWrapper(web_search_requests=queries) if queries is not None else None + return ModelResponse( + usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15, prompt_tokens_details=details) + ) + + +def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) -> tuple[dict, dict]: + sl_object = { + "response_cost": response_cost, + "cache_hit": cache_hit, + "cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost}, + "model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info}, + } + return {"response_cost": response_cost, "standard_logging_object": sl_object}, sl_object + + +@pytest.mark.parametrize( + ("allowance", "units", "total_after", "expected"), + [ + (5000, 5, 5, 5), + (5000, 5, 5000, 5), + (5000, 10, 5005, 5), + (5000, 5, 5010, 0), + (2, 5, 5, 2), + (0, 3, 3, 0), + ], +) +def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, units, total_after, expected): + assert freed_units(allowance, units, total_after) == expected + + +def test_counter_key_is_per_group_and_utc_period(): + assert search_free_tier_key({"count": 1, "period": "month", "shared_group": "g3"}, NOW) == ( + "litellm:search_free_tier:g3:2026-10" + ) + assert search_free_tier_key({"count": 1, "period": "day", "shared_group": "g2"}, NOW) == ( + "litellm:search_free_tier:g2:2026-10-20" + ) + + +@pytest.mark.parametrize( + ("period", "boundary"), + [("month", datetime(2026, 11, 1, tzinfo=timezone.utc)), ("day", datetime(2026, 10, 21, tzinfo=timezone.utc))], +) +def test_counter_ttl_outlives_the_period_boundary_by_under_two_hours(period, boundary): + expiry = NOW + timedelta(seconds=search_free_tier_ttl({"count": 1, "period": period, "shared_group": "g"}, NOW)) + assert boundary < expiry < boundary + timedelta(hours=2) + + +@pytest.mark.asyncio +async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_crossing_call(): + counter = _Counter() + model_info = _model_info("per_query", count=7) + reduced = [] + for _ in range(3): + kwargs, sl_object = _call(model_info, response_cost=1.0) + cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + reduced.append((cost, kwargs["response_cost"], sl_object["response_cost"], sl_object["cost_breakdown"])) + (c1, k1, s1, b1), (c2, k2, s2, b2), (c3, k3, s3, b3) = reduced + assert c1 == k1 == s1 == pytest.approx(1.0 - 5 * RATE) + assert b1["original_cost"] == 1.0 and b1["discount_amount"] == pytest.approx(5 * RATE) + assert b1["total_cost"] == pytest.approx(c1) and b1["tool_usage_cost"] == 0.5 + assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE) + assert b2["discount_amount"] == pytest.approx(2 * RATE) + assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3 + assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 15} + assert counter.ttls["litellm:search_free_tier:test-group:2026-10"] > 0 + + +@pytest.mark.asyncio +async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_query_count(): + counter = _Counter() + model_info = _model_info("per_prompt", count=1, period="day") + kwargs, sl_object = _call(model_info, response_cost=1.0) + first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + kwargs, sl_object = _call(model_info, response_cost=1.0) + second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + assert first == pytest.approx(1.0 - RATE) + assert second == 1.0 + assert counter.totals == {"litellm:search_free_tier:test-group:2026-10-20": 2} + + +@pytest.mark.asyncio +async def test_sidecar_shaped_response_mapping_is_counted_like_a_model_response(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + response = { + "usage": { + "prompt_tokens": 1, + "completion_tokens": 1, + "total_tokens": 2, + "prompt_tokens_details": {"web_search_requests": 3}, + } + } + assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == pytest.approx(1.0 - 3 * RATE) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("counter", [None, _BrokenCounter()]) +async def test_full_price_when_no_counter_is_available(counter): + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0 + assert kwargs["response_cost"] == 1.0 and "discount_amount" not in sl_object["cost_breakdown"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("response", [_response(0), _response(None), ModelResponse()]) +async def test_requests_without_grounding_never_touch_the_counter(response): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, response, counter, NOW) == 1.0 + assert counter.totals == {} + + +@pytest.mark.asyncio +async def test_models_without_an_allowance_never_touch_the_counter(): + counter = _Counter() + model_info = {k: v for k, v in _model_info("per_query", count=100).items() if k != "search_free_requests"} + kwargs, sl_object = _call(model_info, response_cost=1.0) + assert await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) == 1.0 + assert counter.totals == {} + + +@pytest.mark.asyncio +async def test_a_second_callback_run_on_the_same_request_counts_it_once(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=1.0) + first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) + second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW) + assert first == second == pytest.approx(1.0 - 5 * RATE) + assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5} + + +@pytest.mark.asyncio +async def test_a_cache_hit_with_grounded_usage_is_not_counted_and_stays_at_zero(): + counter = _Counter() + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=0.0, cache_hit=True) + assert await apply_search_free_tier(kwargs, sl_object, 0.0, _response(2), counter, NOW) == 0.0 + assert counter.totals == {} + assert "discount_amount" not in sl_object["cost_breakdown"] + + +@pytest.mark.asyncio +async def test_freed_cost_never_exceeds_the_billed_cost(): + counter = _Counter() + billed = 3 * RATE + kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed) + assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0 + assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed) + assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5} diff --git a/tests/unit/test_model_prices_schema.py b/tests/unit/test_model_prices_schema.py index 211a43d2574..86a3a1e6a57 100644 --- a/tests/unit/test_model_prices_schema.py +++ b/tests/unit/test_model_prices_schema.py @@ -10,11 +10,13 @@ from typing import Final import jsonschema import pytest +from pydantic import JsonValue, TypeAdapter import litellm from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts +from litellm.types.utils import SearchFreeRequests REPO_ROOT = Path(__file__).parents[2] GENERATOR_PATH = REPO_ROOT / "ci_cd" / "generate_model_prices_schema.py" @@ -579,3 +581,29 @@ def test_unregistered_provider_guard_flags_only_labels_nobody_registered(): "unknown_root-new_family_models", "vertex_ai-new_family_models", ] + +def _search_free_requests_rows(path: Path) -> Mapping[str, dict[str, JsonValue]]: + prices: Final[dict[str, JsonValue]] = TypeAdapter(dict[str, JsonValue]).validate_json(path.read_bytes()) + return MappingProxyType( + {key: row for key, row in prices.items() if isinstance(row, dict) and "search_free_requests" in row} + ) + + +@pytest.mark.parametrize("path", (PRICES_PATH, BACKUP_PRICES_PATH), ids=("main", "backup")) +def test_search_free_requests_rows_are_grounding_rows_with_one_allowance_per_group(path: Path) -> None: + rows = _search_free_requests_rows(path) + assert rows + allowances = { + key: TypeAdapter(SearchFreeRequests).validate_python(row["search_free_requests"]) for key, row in rows.items() + } + for key, row in rows.items(): + assert row.get("supports_web_search") is True, key + assert row.get("search_context_cost_per_query"), key + assert allowances[key]["count"] > 0, key + for group in {allowance["shared_group"] for allowance in allowances.values()}: + same_group = {(a["count"], a["period"]) for a in allowances.values() if a["shared_group"] == group} + assert len(same_group) == 1, (group, same_group) + + +def test_search_free_requests_backup_rows_match_the_main_catalog() -> None: + assert _search_free_requests_rows(BACKUP_PRICES_PATH) == _search_free_requests_rows(PRICES_PATH) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7db82757e98..63bab893784 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1058,6 +1058,16 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "type": "string", "enum": ["per_prompt", "per_query"], }, + "search_free_requests": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1}, + "period": {"type": "string", "enum": ["day", "month"]}, + "shared_group": {"type": "string"}, + }, + "required": ["count", "period", "shared_group"], + "additionalProperties": False, + }, "citation_cost_per_token": {"type": "number"}, "supported_modalities": { "type": "array",