diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 226996c385f..e36a8bfe82a 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -6,6 +6,7 @@ from pydantic import TypeAdapter EXA_AI_INCLUDED_RESULTS: Final = 10 EXA_AI_ADDITIONAL_RESULT_COST: Final = 0.001 EXA_AI_SUMMARY_COST_PER_PAGE: Final = 0.001 +EXA_AI_TYPES_WITHOUT_SUMMARIES: Final = frozenset({"lite"}) _CONTENTS: Final = TypeAdapter(Mapping[str, object]) @@ -24,6 +25,8 @@ def _requested_results(optional_params: Mapping[str, object]) -> int: def _requests_summaries(optional_params: Mapping[str, object]) -> bool: + if optional_params.get("type") in EXA_AI_TYPES_WITHOUT_SUMMARIES: + return False contents: Final = optional_params.get("contents") return isinstance(contents, Mapping) and _CONTENTS.validate_python(contents).get("summary") not in (None, False) @@ -34,7 +37,7 @@ def exa_ai_list_price(base_price: float, optional_params: Mapping[str, object]) Multi-query searches are joined into a single request, so this prices one request: the search type's base price, a per-result charge above the results that price includes, - and a per-page charge for each requested result when summaries are requested. + and a per-page charge for each requested result when summaries are requested on a type that offers them. """ results: Final = _requested_results(optional_params) additional_results: Final = max(results - EXA_AI_INCLUDED_RESULTS, 0) diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 1c6cf8a86c5..841c9173af9 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -2,9 +2,12 @@ Cost calculation for search providers. """ +import base64 import functools +import hashlib import math from collections.abc import Mapping +from importlib.metadata import PackageNotFoundError, distribution from types import MappingProxyType from typing import Final @@ -30,6 +33,7 @@ SEARCH_MODE_PARAM_BY_PROVIDER: Final[Mapping[str, str]] = MappingProxyType( ) SEARCH_PRICE_FIELDS: Final = ("input_cost_per_query", "tiered_pricing") FLAT_PRICE_ONLY: Final = frozenset({"input_cost_per_query"}) +BUNDLED_COST_MAP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" def _provider_usage( @@ -84,15 +88,32 @@ def _shipped_search_prices() -> Mapping[str, Mapping[str, object]]: ) +@functools.cache +def _bundled_cost_map_edited() -> bool: + """Whether the bundled map's bytes differ from the sha256 its installer recorded, as when an image overwrites the + file. An install that recorded no hash for it, like an editable one, counts as unedited.""" + try: + installed: Final = distribution("litellm").files or () + except PackageNotFoundError: + return False + recorded: Final = next((path.hash for path in installed if str(path) == BUNDLED_COST_MAP_PATH), None) + if recorded is None or recorded.mode != "sha256": + return False + digest: Final = hashlib.sha256(GetModelCostMap.read_local_model_cost_map_bytes()).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode() != recorded.value + + def _own_cost_map_price_fields(entry: str) -> frozenset[str]: - """Price fields a cost map served from the operator's own URL sets differently from the map LiteLLM ships, so a - mirror of LiteLLM's map prices nothing.""" + """Price fields the operator's own cost map sets. A map served from their own URL counts only where it differs + from the map LiteLLM ships, so a mirror of LiteLLM's map prices nothing, and a bundled map their image edited + counts in full, since nothing is left to compare it with.""" source: Final = get_model_cost_map_source_info() - if source["source"] != "remote" or source["url"] == DEFAULT_MODEL_COST_MAP_URL: + loaded: Final = GetModelCostMap.loaded_model_cost_map().get(entry) + if source["source"] != "remote": + return _changed_price_fields(loaded, None) if _bundled_cost_map_edited() else frozenset() + if source["url"] == DEFAULT_MODEL_COST_MAP_URL: return frozenset() - return _changed_price_fields( - GetModelCostMap.loaded_model_cost_map().get(entry), _shipped_search_prices().get(entry) - ) + return _changed_price_fields(loaded, _shipped_search_prices().get(entry)) def _operator_price_fields(entry: str) -> frozenset[str]: @@ -104,13 +125,14 @@ def _registered_at_runtime(entry: str) -> bool: def _operator_priced(entry: str) -> bool: - """Whether an operator set a price on `entry`, with `register_model` or a cost map served from their own URL.""" + """Whether an operator set a price on `entry`, with `register_model` or their own cost map.""" return bool(_operator_price_fields(entry)) def _operator_flat_price_replaces_tiers(entry: str) -> bool: - """Whether the operator set `entry`'s flat price but not its tiers, so the tiers LiteLLM ships stop applying.""" - return _operator_price_fields(entry) == FLAT_PRICE_ONLY + """Whether the operator's latest price on `entry` is a flat price without tiers, so the tiers already on the entry + stop applying. `register_model` runs after the cost map loads, so its prices outrank the operator's own map.""" + return (_runtime_price_fields(entry) or _own_cost_map_price_fields(entry)) == FLAT_PRICE_ONLY def _exa_reported_total(response: object) -> float | None: @@ -155,7 +177,7 @@ def search_provider_cost_per_query( Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the provider's mode parameter (Tavily `search_depth`, Linkup `depth`, Exa `type`). Exa bills one request at the `costDollars.total` it reports, else at its list price, unless an operator priced the entry. A flat price - an operator set over the tiers LiteLLM ships replaces them. + an operator set last, over tiers LiteLLM or their own cost map ships, replaces those tiers. Args: model: Model name (e.g., "exa_ai/search", "tavily/search") diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 11d84f61612..8866dbbab05 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -559,6 +559,17 @@ def test_exa_estimate_adds_each_summarized_page(contents: object, summarized_pag ) +@pytest.mark.usefixtures("local_model_cost_map") +def test_exa_estimate_adds_no_summary_charge_on_lite() -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + lite: Final = {"type": "lite", "max_results": 12} + # https://exa.ai/docs/reference/pricing (2026-10-28) lists no AI page summary price for Lite + assert search_provider_cost_per_query( + "exa_ai/search", "exa_ai", optional_params={**lite, "contents": {"summary": True}} + ) == search_provider_cost_per_query("exa_ai/search", "exa_ai", optional_params=lite) + + @pytest.mark.usefixtures("local_model_cost_map") def test_exa_estimate_prices_one_request_however_many_queries_it_joins() -> None: from litellm.search.cost_calculator import search_provider_cost_per_query diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index c64341e596b..c46cd9fb746 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -1,5 +1,8 @@ +import base64 import datetime +import hashlib import json +from pathlib import Path from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass from typing import Final @@ -12,7 +15,7 @@ import litellm from litellm.constants import DEFAULT_MODEL_COST_MAP_URL from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.search.cost_calculator import search_provider_cost_per_query +from litellm.search.cost_calculator import _bundled_cost_map_edited, search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" @@ -419,9 +422,89 @@ def test_an_exa_price_from_the_operators_own_cost_map_wins_over_the_cost_exa_rep assert response._hidden_params["response_cost"] == pytest.approx(expected) +def _sha256_record(body: bytes) -> str: + return base64.urlsafe_b64encode(hashlib.sha256(body).digest()).rstrip(b"=").decode() + + +@pytest.fixture +def _installed_litellm(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Iterator[Callable[[str], None]]: + def install(recorded_sha256: str) -> None: + dist_info: Final = tmp_path / "litellm-0.0.0.dist-info" + dist_info.mkdir() + (dist_info / "METADATA").write_text("Metadata-Version: 2.1\nName: litellm\nVersion: 0.0.0\n") + (dist_info / "RECORD").write_text( + f"litellm/model_prices_and_context_window_backup.json,sha256={recorded_sha256},1\n" + ) + (tmp_path / "litellm").mkdir() + (tmp_path / "litellm" / "model_prices_and_context_window_backup.json").touch() + monkeypatch.syspath_prepend(str(tmp_path)) + _bundled_cost_map_edited.cache_clear() + + yield install + _bundled_cost_map_edited.cache_clear() + + +@pytest.mark.usefixtures("_loaded_catalog_prices") +@pytest.mark.parametrize("bundled_map_edited", [False, True], ids=["bundled-map-as-installed", "bundled-map-edited"]) +def test_a_bundled_cost_map_edited_after_install_prices_exa_from_the_map( + respx_mock: respx.MockRouter, _installed_litellm: Callable[[str], None], bundled_map_edited: bool +) -> None: + bundled: Final = GetModelCostMap.read_local_model_cost_map_bytes() + _installed_litellm(_sha256_record(bundled + b" " if bundled_map_edited else bundled)) + respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_TOTAL}}) + tiers: Final = GetModelCostMap.loaded_model_cost_map()["exa_ai/search"]["tiered_pricing"] + assert isinstance(tiers, list) and tiers + map_price: Final = next( + tier["input_cost_per_query"] + for tier in tiers + if tier["max_results_range"][0] <= 10 <= tier["max_results_range"][1] + ) + + response: Final = litellm.search(query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key") + + assert response._hidden_params["response_cost"] == pytest.approx( + map_price if bundled_map_edited else EXA_REPORTED_TOTAL + ) + + REGISTERED_TIER_PRICE: Final = 0.0456 +@pytest.mark.parametrize( + ("own_map_price", "registration", "expected"), + [ + pytest.param( + {"tiered_pricing": [{"input_cost_per_query": OPERATOR_TIER_PRICE, "max_results_range": [0, 100]}]}, + {"input_cost_per_query": EXA_REGISTERED_PRICE}, + EXA_REGISTERED_PRICE, + id="flat-registered-over-own-map-tiers", + ), + pytest.param( + {"input_cost_per_query": OPERATOR_FLAT_PRICE}, + {"tiered_pricing": [{"input_cost_per_query": REGISTERED_TIER_PRICE, "max_results_range": [0, 100]}]}, + REGISTERED_TIER_PRICE, + id="tiers-registered-over-own-map-flat-price", + ), + ], +) +def test_a_price_registered_over_the_operators_own_cost_map_bills_the_registered_price( + respx_mock: respx.MockRouter, + _remote_cost_map: Callable[[str, ServedExaEntry], None], + own_map_price: Mapping[str, object], + registration: Mapping[str, object], + expected: float, +) -> None: + _remote_cost_map( + OPERATOR_COST_MAP_URL, lambda _: {**own_map_price, "litellm_provider": "exa_ai", "mode": "search"} + ) + litellm.register_model({"exa_ai/search": dict(registration)}) + respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_TOTAL}}) + + response: Final = litellm.search(query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key") + + assert response._hidden_params["response_cost"] == pytest.approx(expected) + + @pytest.mark.usefixtures("_loaded_catalog_prices") @pytest.mark.parametrize( ("registration", "expected"),