From 00c449ad7fb648c22d1327b26081f1cd3572ed62 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 16:31:48 -0700 Subject: [PATCH 1/8] fix(exa): bill the search cost Exa reports and keep Exa's result fields The Exa adapter hands each reply's costDollars to search cost tracking, which bills Exa's reported total as the request's base cost, the way Parallel AI's reported usage already prices its searches. Cost discounts and margins still apply on top, an input_cost_per_query set on the search tool or the call still wins, and a reply without a usable total keeps the cost map price. Results now keep every field Exa returns (id, author, image, summary, ...) instead of only the shared ones. --- litellm/llms/exa_ai/search/cost_calculator.py | 22 +++ litellm/llms/exa_ai/search/transformation.py | 58 ++++--- litellm/search/cost_calculator.py | 12 +- .../sdk/test_exa_search_envelope_sdk.py | 1 + .../llms/exa_ai/search/test_transformation.py | 158 +++++++++++++++++- tests/unit/search/test_main.py | 70 ++++++++ 6 files changed, 290 insertions(+), 31 deletions(-) create mode 100644 litellm/llms/exa_ai/search/cost_calculator.py diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py new file mode 100644 index 00000000000..007cac9b68f --- /dev/null +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -0,0 +1,22 @@ +from collections.abc import Mapping +from typing import Annotated, Final + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +from litellm.constants import INTERNAL_KWARG_PREFIX + +EXA_AI_COST_DOLLARS_PARAM: Final = f"{INTERNAL_KWARG_PREFIX}exa_ai_cost_dollars" + + +class _ExaCostDollars(BaseModel): + model_config = ConfigDict(frozen=True) + + total: Annotated[float, Field(strict=True, ge=0, allow_inf_nan=False)] + + +def exa_ai_reported_cost(optional_params: Mapping[str, object]) -> float | None: + """The `costDollars.total` Exa reported for the request, when it is a finite non-negative number.""" + try: + return _ExaCostDollars.model_validate(optional_params.get(EXA_AI_COST_DOLLARS_PARAM)).total + except ValidationError: + return None diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 73e341d0033..574217ef3d8 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -17,12 +17,13 @@ from litellm.llms.base_llm.search.transformation import ( SearchResponse, SearchResult, ) +from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM from litellm.secret_managers.main import get_secret_str _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True)) _STRS: Final = TypeAdapter(Iterable[str], config=ConfigDict(hide_input_in_errors=True)) -_RESULT_EXTRAS: Final = (("highlights", "highlights"), ("highlightScores", "highlight_scores"), ("score", "score")) +_RENAMED_RESULT_FIELDS: Final = frozenset({"text", "publishedDate", "highlightScores"}) class _ExaAISearchRequestRequired(TypedDict): @@ -171,9 +172,11 @@ class ExaAISearchConfig(BaseSearchConfig): - results[].summary → SearchResult.snippet (fallback when "text" and "highlights" are absent) - results[].publishedDate → SearchResult.date - No last_updated field in Exa AI response (set to None) - - results[].highlights / highlightScores / score → SearchResult.highlights / highlight_scores / score, - each set only when Exa returned it + - results[].highlightScores → SearchResult.highlight_scores, set only when Exa returned it + - every other results[] field Exa returned (id, author, image, summary, highlights, score, ...) + → the SearchResult as is - every top-level key except results (requestId, costDollars, ...) → SearchResponse.provider_metadata + - costDollars → the logging object's optional_params, where search cost tracking reads Exa's reported total Args: raw_response: Raw httpx response from Exa AI API @@ -183,30 +186,35 @@ class ExaAISearchConfig(BaseSearchConfig): SearchResponse with standardized format """ response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) - - # Transform results to SearchResult objects - results: Final = [] - for raw_result in _JSON_VALUES.validate_python(response_json.get("results", [])): - result = _JSON_OBJECT.validate_python(raw_result) - search_result = SearchResult.model_validate( - { - "title": result.get("title", ""), - "url": result.get("url", ""), - "snippet": result.get("text") - or "\n\n".join(_STRS.validate_python(result.get("highlights") or [])) - or result.get("summary") - or "", - "date": result.get("publishedDate"), # ISO 8601 datetime string - "last_updated": None, # Exa AI doesn't provide last_updated in response - **{field: result[key] for key, field in _RESULT_EXTRAS if key in result}, - } - ) - results.append(search_result) - - return SearchResponse.model_validate( + response: Final = SearchResponse.model_validate( { - "results": results, + "results": [ + _transform_result(value) for value in _JSON_VALUES.validate_python(response_json.get("results", [])) + ], "object": "search", "provider_metadata": {key: value for key, value in response_json.items() if key != "results"}, } ) + logging_obj.optional_params = { # rebind-ok: the logging object is the adapter's only channel to cost tracking + **logging_obj.optional_params, + EXA_AI_COST_DOLLARS_PARAM: response_json.get("costDollars"), + } + return response + + +def _transform_result(raw_result: object) -> SearchResult: + result: Final = _JSON_OBJECT.validate_python(raw_result) + return SearchResult.model_validate( + { + **{key: value for key, value in result.items() if key not in _RENAMED_RESULT_FIELDS}, + "title": result.get("title", ""), + "url": result.get("url", ""), + "snippet": result.get("text") + or "\n\n".join(_STRS.validate_python(result.get("highlights") or [])) + or result.get("summary") + or "", + "date": result.get("publishedDate"), + "last_updated": None, + **({"highlight_scores": result["highlightScores"]} if "highlightScores" in result else {}), + } + ) diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 15755128136..22c58b3efbc 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -10,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.llms.exa_ai.search.cost_calculator import exa_ai_reported_cost from litellm.utils import get_model_info PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( @@ -70,7 +71,8 @@ def search_provider_cost_per_query( Returns (input_cost, output_cost) where input_cost = queries * cost_per_query Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the - provider's mode parameter (Tavily `search_depth`, Linkup `depth`). + provider's mode parameter (Tavily `search_depth`, Linkup `depth`). An Exa request is billed the + `costDollars.total` Exa reported for it, when there is one. Args: model: Model name (e.g., "exa_ai/search", "tavily/search") @@ -93,6 +95,14 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) + reported_cost: Final = ( + exa_ai_reported_cost(optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS) + if custom_llm_provider == "exa_ai" + else None + ) + if reported_cost is not None: + return (reported_cost, 0.0) + model_info: Final = get_model_info( model=_pricing_model(model, custom_llm_provider, optional_params), custom_llm_provider=custom_llm_provider, diff --git a/tests/integration/sdk/test_exa_search_envelope_sdk.py b/tests/integration/sdk/test_exa_search_envelope_sdk.py index 73c3538fa45..3ab2a1a4969 100644 --- a/tests/integration/sdk/test_exa_search_envelope_sdk.py +++ b/tests/integration/sdk/test_exa_search_envelope_sdk.py @@ -30,6 +30,7 @@ _EXPECTED: Final[dict[str, JsonValue]] = { "object": "search", "results": [ { + "id": "https://result.invalid/sdk", "title": "SDK result", "url": "https://result.invalid/sdk", "snippet": "First line.\n\nSecond line.", diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index a5a5d5e22dc..6a450e9efaa 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -1,17 +1,26 @@ +import asyncio import json +import uuid +from collections.abc import Mapping from typing import Final from unittest.mock import Mock import httpx import pytest import respx -from pydantic import ValidationError +from pydantic import TypeAdapter, ValidationError import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult +from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM, exa_ai_reported_cost from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig +from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" +EXA_REPORTED_COST: Final = 0.022 +_LOGGED: Final = TypeAdapter(Mapping[str, object]) @pytest.fixture @@ -41,7 +50,7 @@ def test_transform_search_response_snippet_falls_back_through_content_modes( json={"results": [{"title": "Title", "url": "https://example.com", **content_fields}]}, ) - response: Final = ExaAISearchConfig().transform_search_response(raw_response, logging_obj=Mock()) + response: Final = ExaAISearchConfig().transform_search_response(raw_response, logging_obj=Mock(optional_params={})) assert response.results[0].snippet == expected_snippet @@ -93,7 +102,9 @@ def test_search_sends_max_results_as_num_results(respx_mock: respx.MockRouter) - def _transform(payload: object) -> SearchResponse: - return ExaAISearchConfig().transform_search_response(httpx.Response(200, json=payload), logging_obj=Mock()) + return ExaAISearchConfig().transform_search_response( + httpx.Response(200, json=payload), logging_obj=Mock(optional_params={}) + ) def test_transform_search_response_maps_every_exa_result_in_provider_order(): @@ -125,13 +136,16 @@ def test_transform_search_response_maps_every_exa_result_in_provider_order(): assert response.results == [ SearchResult.model_validate( { + "id": "https://example.com/first", "title": "First", "url": "https://example.com/first", "snippet": "First text", "date": "2024-01-05T00:00:00.000Z", + "author": "Ada", "highlights": ["ignored when text is present"], "highlight_scores": [0.5], "score": 0.42, + "image": None, } ), SearchResult.model_validate( @@ -142,7 +156,9 @@ def test_transform_search_response_maps_every_exa_result_in_provider_order(): "highlights": ["one", "two"], } ), - SearchResult(title="Third", url="https://example.com/third", snippet="Summary"), + SearchResult.model_validate( + {"title": "Third", "url": "https://example.com/third", "snippet": "Summary", "summary": "Summary"} + ), ] assert response.model_extra == { "provider_metadata": { @@ -162,7 +178,15 @@ def test_transform_search_response_maps_every_exa_result_in_provider_order(): ({"highlightScores": [0.1]}, {"highlight_scores": [0.1]}), ({"score": None}, {"score": None}), ({"highlights": [], "highlightScores": [], "score": 0}, {"highlights": [], "highlight_scores": [], "score": 0}), - ({"id": "https://example.com", "author": "Ada", "image": None, "summary": "a summary"}, {}), + ({"publishedDate": "2024-01-05T00:00:00.000Z"}, {}), + ( + {"id": "https://example.com", "author": "Ada", "image": None, "summary": "a summary"}, + {"id": "https://example.com", "author": "Ada", "image": None, "summary": "a summary"}, + ), + ( + {"favicon": "https://example.com/favicon.ico", "subpages": []}, + {"favicon": "https://example.com/favicon.ico", "subpages": []}, + ), ], ) def test_transform_search_response_attaches_only_the_exa_result_fields_it_returned( @@ -283,3 +307,127 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str) -> ExaAISearchConfig().get_complete_url(api_base=api_base, optional_params={}) == "https://gateway.example.com/exa/search" ) + + +def _catalog_price(max_results: int) -> float: + price, _ = search_provider_cost_per_query( + model="exa_ai/search", custom_llm_provider="exa_ai", optional_params={"max_results": max_results} + ) + return price + + +@pytest.mark.usefixtures("exa_api_key") +@pytest.mark.parametrize("reported_cost", [EXA_REPORTED_COST, 0, 3]) +def test_search_bills_the_cost_exa_reports(respx_mock: respx.MockRouter, reported_cost: float) -> None: + respx_mock.post(EXA_SEARCH_URL).respond( + json={"results": [], "costDollars": {"total": reported_cost, "search": {"neural": reported_cost}}} + ) + assert _catalog_price(25) != reported_cost + + response: Final = litellm.search(query="asyncio TaskGroup", search_provider="exa_ai", max_results=25) + + assert response.hidden_params["response_cost"] == reported_cost + + +@pytest.mark.usefixtures("exa_api_key") +@pytest.mark.parametrize( + "body", + [ + '{"results": []}', + '{"results": [], "costDollars": {}}', + '{"results": [], "costDollars": {"total": null}}', + '{"results": [], "costDollars": {"total": -0.01}}', + '{"results": [], "costDollars": {"total": "0.022"}}', + '{"results": [], "costDollars": {"total": true}}', + '{"results": [], "costDollars": {"total": NaN}}', + '{"results": [], "costDollars": {"total": Infinity}}', + '{"results": [], "costDollars": "0.022"}', + '{"results": [], "costDollars": [0.022]}', + ], +) +def test_search_without_a_usable_exa_cost_bills_the_catalog_price(respx_mock: respx.MockRouter, body: str) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(content=body, headers={"content-type": "application/json"}) + + response: Final = litellm.search(query="asyncio TaskGroup", search_provider="exa_ai", max_results=25) + + assert response.hidden_params["response_cost"] == pytest.approx(_catalog_price(25)) + + +@pytest.mark.usefixtures("exa_api_key") +@pytest.mark.parametrize( + ("config", "expected_cost"), + [("cost_discount_config", EXA_REPORTED_COST * 0.9), ("cost_margin_config", EXA_REPORTED_COST * 1.1)], +) +def test_an_exa_discount_or_margin_applies_on_top_of_the_cost_exa_reports( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, config: str, expected_cost: float +) -> None: + monkeypatch.setattr(litellm, config, {"exa_ai": 0.1}) + respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}) + + response: Final = litellm.search(query="asyncio TaskGroup", search_provider="exa_ai", max_results=25) + + assert response.hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("exa_api_key") +def test_a_caller_cannot_set_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(EXA_SEARCH_URL).respond(json={"results": []}) + + response: Final = litellm.search( + query="asyncio TaskGroup", + search_provider="exa_ai", + max_results=25, + **{EXA_AI_COST_DOLLARS_PARAM: {"total": 0}}, + ) + + assert EXA_AI_COST_DOLLARS_PARAM not in json.loads(route.calls.last.request.content) + assert response.hidden_params["response_cost"] == pytest.approx(_catalog_price(25)) + + +def test_a_reply_without_a_cost_does_not_keep_an_earlier_replys_cost() -> None: + logging_obj: Final = Mock(optional_params={}) + config: Final = ExaAISearchConfig() + + config.transform_search_response( + httpx.Response(200, json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}), logging_obj=logging_obj + ) + assert exa_ai_reported_cost(logging_obj.optional_params) == EXA_REPORTED_COST + config.transform_search_response(httpx.Response(200, json={"results": []}), logging_obj=logging_obj) + + assert exa_ai_reported_cost(logging_obj.optional_params) is None + + +class _SpendCapture(CustomLogger): + def __init__(self, call_id: str) -> None: + super().__init__() + self.call_id: Final = call_id + self.costs: Final[list[object]] = [] # mutable-ok: the logging worker appends one entry per logged call + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + if kwargs.get("litellm_call_id") == self.call_id: + self.costs.append(_LOGGED.validate_python(kwargs.get("standard_logging_object") or {}).get("response_cost")) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("exa_api_key") +async def test_async_search_logs_the_cost_exa_reports_as_spend( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + GLOBAL_LOGGING_WORKER.start() + capture: Final = _SpendCapture(call_id=f"exa-{uuid.uuid4()}") + monkeypatch.setattr(litellm, "callbacks", [capture]) + respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}) + assert _catalog_price(25) != EXA_REPORTED_COST + + response: Final = await litellm.asearch( + query="asyncio TaskGroup", search_provider="exa_ai", max_results=25, litellm_call_id=capture.call_id + ) + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert response.hidden_params["response_cost"] == EXA_REPORTED_COST + assert capture.costs == [EXA_REPORTED_COST] diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index e30fd2eb6f8..7f22c4fe398 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -15,6 +15,11 @@ from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" LINKUP_SEARCH_URL: Final = "https://api.linkup.so/v1/search" +EXA_REPORTED_COST: Final = 0.022 +EXA_REPLY_WITH_COST: Final = { + "results": [{"title": "t", "url": "https://example.com"}], + "costDollars": {"total": EXA_REPORTED_COST}, +} SEARCH_MODE_PRICES: Final = { "tavily/search": 0.002, "tavily/search-advanced": 0.007, @@ -115,6 +120,71 @@ def test_unpriced_fallback_search_tool_is_billed_at_its_own_price_not_the_failed assert fallback._hidden_params["response_cost"] == pytest.approx(map_price) +@pytest.mark.usefixtures("_local_model_cost_map") +def test_a_search_tool_price_overrides_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + configured_price: Final = 0.05 + assert configured_price != EXA_REPORTED_COST + + reported: Final = litellm.search( + query="litellm release notes", + search_provider="exa_ai", + api_key="test-exa-key", + litellm_logging_obj=_proxy_style_logging_obj(), + ) + tool_priced: Final = litellm.search( + query="litellm release notes", + search_provider="exa_ai", + api_key="test-exa-key", + input_cost_per_query=configured_price, + litellm_logging_obj=_proxy_style_logging_obj(), + ) + + assert reported.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) + assert tool_priced.hidden_params["response_cost"] == pytest.approx(configured_price) + + +@pytest.mark.usefixtures("_local_model_cost_map") +def test_unpriced_fallback_search_tool_bills_the_cost_exa_reports_not_the_failed_primarys_price( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(EXA_SEARCH_URL).mock( + side_effect=(httpx.Response(500, json={"error": "primary down"}), httpx.Response(200, json=EXA_REPLY_WITH_COST)) + ) + primary_price: Final = 0.05 + shared_by_every_fallback_attempt: Final = _proxy_style_logging_obj() + + with pytest.raises(litellm.InternalServerError): + litellm.search( + query="litellm release notes", + search_provider="exa_ai", + api_key="test-exa-key", + input_cost_per_query=primary_price, + litellm_logging_obj=shared_by_every_fallback_attempt, + ) + fallback: Final = litellm.search( + query="litellm release notes", + search_provider="exa_ai", + api_key="test-exa-key", + litellm_logging_obj=shared_by_every_fallback_attempt, + ) + + assert fallback.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) + + +@pytest.mark.usefixtures("_local_model_cost_map") +def test_the_cost_exa_reports_wins_over_a_price_registered_in_the_cost_map(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + registered_price: Final = 0.123 + litellm.register_model( + {"exa_ai/search": {"input_cost_per_query": registered_price, "litellm_provider": "exa_ai", "mode": "search"}} + ) + + response: Final = litellm.search(query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key") + + assert response.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) + + @pytest.mark.usefixtures("_search_mode_prices") @pytest.mark.parametrize( ("provider", "optional_params", "expected"), From 527933dd2ef603d55e6452beb3983f6e0e048073 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 17:20:16 -0700 Subject: [PATCH 2/8] fix(exa): let a flat Exa price in the cost map override the reported cost LiteLLM ships exa_ai/search with tiers only, kept for older releases that read the same price list. A flat input_cost_per_query on that entry, from register_model(), an admin's own cost map URL or an edited bundled file, is an override, so it is billed instead of Exa's reported cost and beats the tiers. Exa's reported cost is still billed when the cost map has no Exa entry at all. --- litellm/search/cost_calculator.py | 23 +++++++++++++-- tests/unit/search/test_main.py | 49 +++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 22c58b3efbc..0d6e2f04071 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -17,6 +17,7 @@ PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = T tuple[Mapping[str, object], ...] ) EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) +PRICE_ENTRY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) SEARCH_MODE_PARAM_BY_PROVIDER: Final[Mapping[str, str]] = MappingProxyType( {"tavily": "search_depth", "linkup": "depth"} ) @@ -34,6 +35,18 @@ def _provider_usage( return None +def _generic_search_entry(model: str, custom_llm_provider: str | None) -> str: + generic_model: Final = f"{custom_llm_provider}/search" + return generic_model if model == f"{custom_llm_provider}/{generic_model}" else model + + +def _has_flat_price(entry: str) -> bool: + live: Final[object] = litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed + return ( + isinstance(live, Mapping) and PRICE_ENTRY_ADAPTER.validate_python(live).get("input_cost_per_query") is not None + ) + + def _registered_at_runtime(entry: str) -> bool: """Whether `entry`'s live price differs from the catalog as loaded, i.e. `register_model` changed it.""" live: Final[object] = litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed @@ -72,7 +85,8 @@ def search_provider_cost_per_query( Returns (input_cost, output_cost) where input_cost = queries * cost_per_query Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the provider's mode parameter (Tavily `search_depth`, Linkup `depth`). An Exa request is billed the - `costDollars.total` Exa reported for it, when there is one. + `costDollars.total` Exa reported for it, unless its price entry has an `input_cost_per_query`. LiteLLM ships + Exa's entry with tiers only (what older releases bill), so a flat price there is always someone's override. Args: model: Model name (e.g., "exa_ai/search", "tavily/search") @@ -95,9 +109,12 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) + exa_price_set: Final = custom_llm_provider == "exa_ai" and _has_flat_price( + _generic_search_entry(model, custom_llm_provider) + ) reported_cost: Final = ( exa_ai_reported_cost(optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS) - if custom_llm_provider == "exa_ai" + if custom_llm_provider == "exa_ai" and not exa_price_set else None ) if reported_cost is not None: @@ -110,7 +127,7 @@ def search_provider_cost_per_query( # Check for tiered pricing (e.g., Exa AI based on max_results) tiered_pricing: Final = model_info.get("tiered_pricing") - if tiered_pricing and isinstance(tiered_pricing, list): + if tiered_pricing and isinstance(tiered_pricing, list) and not exa_price_set: max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results cost_per_query = 0.0 diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index 7f22c4fe398..3428c3097e4 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -1,6 +1,6 @@ import datetime import json -from collections.abc import Mapping +from collections.abc import Callable, Mapping from typing import Final import httpx @@ -172,14 +172,51 @@ def test_unpriced_fallback_search_tool_bills_the_cost_exa_reports_not_the_failed assert fallback.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) -@pytest.mark.usefixtures("_local_model_cost_map") -def test_the_cost_exa_reports_wins_over_a_price_registered_in_the_cost_map(respx_mock: respx.MockRouter) -> None: - respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) - registered_price: Final = 0.123 +def _register_exa_price(monkeypatch: pytest.MonkeyPatch, price: float) -> None: litellm.register_model( - {"exa_ai/search": {"input_cost_per_query": registered_price, "litellm_provider": "exa_ai", "mode": "search"}} + {"exa_ai/search": {"input_cost_per_query": price, "litellm_provider": "exa_ai", "mode": "search"}} ) + +def _serve_exa_price_in_cost_map(monkeypatch: pytest.MonkeyPatch, price: float) -> None: + monkeypatch.setitem( + litellm.model_cost, "exa_ai/search", {**litellm.model_cost["exa_ai/search"], "input_cost_per_query": price} + ) + + +@pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize("set_exa_price", [_register_exa_price, _serve_exa_price_in_cost_map]) +@pytest.mark.parametrize( + "exa_reply", [EXA_REPLY_WITH_COST, {"results": []}], ids=["exa-reports-a-cost", "exa-reports-no-cost"] +) +def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_the_tiers( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + set_exa_price: Callable[[pytest.MonkeyPatch, float], None], + exa_reply: Mapping[str, object], +) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=exa_reply) + tier_price, _ = search_provider_cost_per_query( + model="exa_ai/search", custom_llm_provider="exa_ai", optional_params={"max_results": 25} + ) + set_price: Final = 0.003 + assert set_price not in (tier_price, EXA_REPORTED_COST) + set_exa_price(monkeypatch, set_price) + + response: Final = litellm.search( + query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key", max_results=25 + ) + + assert response.hidden_params["response_cost"] == pytest.approx(set_price) + + +@pytest.mark.usefixtures("_local_model_cost_map") +def test_the_cost_exa_reports_is_billed_when_the_cost_map_has_no_exa_entry( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + monkeypatch.delitem(litellm.model_cost, "exa_ai/search") + response: Final = litellm.search(query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key") assert response.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) From 6ae3e9751e977146630466a33821059cec4f8154 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 17:39:06 -0700 Subject: [PATCH 3/8] fix(exa): treat any Exa price someone set as an override, per search type too LiteLLM's exa_ai/search entry keeps fixed legacy tiers that only older releases bill. Tiers that differ from them, prices register_model() set even at the same values, a flat price, or a row for one search type (exa_ai/search-lite, ...) are someone's override and replace the cost Exa reports. Any search entry's flat price now beats its tiers, since LiteLLM never ships both on one entry. Exa's reported cost now rides on the reply's hidden params, so completion_cost() and cost_per_token() called on an Exa reply return it too, and a reply with "results": null comes back with no results. --- litellm/cost_calculator.py | 7 +- litellm/llms/exa_ai/search/cost_calculator.py | 45 ++++++++- litellm/llms/exa_ai/search/transformation.py | 11 +-- litellm/search/cost_calculator.py | 32 +++---- .../exa_ai/search/test_cost_calculator.py | 40 ++++++++ .../llms/exa_ai/search/test_transformation.py | 32 ++++--- tests/unit/search/test_main.py | 91 ++++++++++++++++++- 7 files changed, 211 insertions(+), 47 deletions(-) create mode 100644 tests/unit/llms/exa_ai/search/test_cost_calculator.py diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 86b22069a39..4c46ab34468 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -25,7 +25,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping from litellm.litellm_core_utils.cost_discount import resolve_cost_discount from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type -from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, get_hidden_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) @@ -1913,7 +1913,10 @@ def completion_cost( model=search_model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries, - optional_params=optional_params, + optional_params={ + **(get_hidden_params(completion_response) or {}), + **(optional_params or {}), + }, ) ) diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 007cac9b68f..5d6e38d5bad 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -1,11 +1,14 @@ from collections.abc import Mapping from typing import Annotated, Final -from pydantic import BaseModel, ConfigDict, Field, ValidationError +from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError from litellm.constants import INTERNAL_KWARG_PREFIX +from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry EXA_AI_COST_DOLLARS_PARAM: Final = f"{INTERNAL_KWARG_PREFIX}exa_ai_cost_dollars" +_PRICE_FIELDS: Final = frozenset({"input_cost_per_query", "tiered_pricing"}) +_ENTRY: Final = TypeAdapter(Mapping[str, object]) class _ExaCostDollars(BaseModel): @@ -14,9 +17,45 @@ class _ExaCostDollars(BaseModel): total: Annotated[float, Field(strict=True, ge=0, allow_inf_nan=False)] -def exa_ai_reported_cost(optional_params: Mapping[str, object]) -> float | None: +class ExaAITier(BaseModel): + model_config = ConfigDict(frozen=True) + + input_cost_per_query: float + max_results_range: tuple[int, int] + + +_TIERS: Final = TypeAdapter(tuple[ExaAITier, ...]) + +EXA_AI_LEGACY_TIERS: Final = ( + ExaAITier(input_cost_per_query=0.005, max_results_range=(0, 25)), + ExaAITier(input_cost_per_query=0.025, max_results_range=(26, 100)), +) +"""The tiers LiteLLM ships on `exa_ai/search`. Only older releases bill them, so they never change.""" + + +def exa_ai_reported_cost(params: Mapping[str, object]) -> float | None: """The `costDollars.total` Exa reported for the request, when it is a finite non-negative number.""" try: - return _ExaCostDollars.model_validate(optional_params.get(EXA_AI_COST_DOLLARS_PARAM)).total + return _ExaCostDollars.model_validate(params.get(EXA_AI_COST_DOLLARS_PARAM)).total except ValidationError: return None + + +def _legacy_tiers(tiers: object) -> bool: + try: + return _TIERS.validate_python(tiers) == EXA_AI_LEGACY_TIERS + except ValidationError: + return False + + +def exa_ai_price_set(entry: object) -> bool: + """Whether an Exa price entry holds a price someone set, which replaces the cost Exa reports. LiteLLM ships + Exa with its legacy tiers only, so anything else there was set by `register_model()`, an admin's own cost map, + or an edited bundled file.""" + if isinstance(entry, AliasedCostEntry) and entry.registered_fields & _PRICE_FIELDS: + return True + if not isinstance(entry, Mapping): + return False + fields: Final = _ENTRY.validate_python(entry) + tiers: Final = fields.get("tiered_pricing") + return fields.get("input_cost_per_query") is not None or (tiers is not None and not _legacy_tiers(tiers)) diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 574217ef3d8..f4969bc1536 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -10,6 +10,7 @@ from typing import Final, TypedDict import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( @@ -176,7 +177,7 @@ class ExaAISearchConfig(BaseSearchConfig): - every other results[] field Exa returned (id, author, image, summary, highlights, score, ...) → the SearchResult as is - every top-level key except results (requestId, costDollars, ...) → SearchResponse.provider_metadata - - costDollars → the logging object's optional_params, where search cost tracking reads Exa's reported total + - costDollars → the response's hidden params, where search cost tracking reads Exa's reported total Args: raw_response: Raw httpx response from Exa AI API @@ -189,16 +190,14 @@ class ExaAISearchConfig(BaseSearchConfig): response: Final = SearchResponse.model_validate( { "results": [ - _transform_result(value) for value in _JSON_VALUES.validate_python(response_json.get("results", [])) + _transform_result(value) + for value in _JSON_VALUES.validate_python(response_json.get("results") or []) ], "object": "search", "provider_metadata": {key: value for key, value in response_json.items() if key != "results"}, } ) - logging_obj.optional_params = { # rebind-ok: the logging object is the adapter's only channel to cost tracking - **logging_obj.optional_params, - EXA_AI_COST_DOLLARS_PARAM: response_json.get("costDollars"), - } + set_hidden_params(response, {EXA_AI_COST_DOLLARS_PARAM: response_json.get("costDollars")}) return response diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 0d6e2f04071..5eedd74d783 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -10,16 +10,15 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.llms.exa_ai.search.cost_calculator import exa_ai_reported_cost +from litellm.llms.exa_ai.search.cost_calculator import exa_ai_price_set, exa_ai_reported_cost from litellm.utils import get_model_info PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) -PRICE_ENTRY_ADAPTER: Final = TypeAdapter(Mapping[str, object]) SEARCH_MODE_PARAM_BY_PROVIDER: Final[Mapping[str, str]] = MappingProxyType( - {"tavily": "search_depth", "linkup": "depth"} + {"tavily": "search_depth", "linkup": "depth", "exa_ai": "type"} ) @@ -40,11 +39,8 @@ def _generic_search_entry(model: str, custom_llm_provider: str | None) -> str: return generic_model if model == f"{custom_llm_provider}/{generic_model}" else model -def _has_flat_price(entry: str) -> bool: - live: Final[object] = litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed - return ( - isinstance(live, Mapping) and PRICE_ENTRY_ADAPTER.validate_python(live).get("input_cost_per_query") is not None - ) +def _live_entry(entry: str) -> object: + return litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed def _registered_at_runtime(entry: str) -> bool: @@ -84,9 +80,9 @@ def search_provider_cost_per_query( Returns (input_cost, output_cost) where input_cost = queries * cost_per_query Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the - provider's mode parameter (Tavily `search_depth`, Linkup `depth`). An Exa request is billed the - `costDollars.total` Exa reported for it, unless its price entry has an `input_cost_per_query`. LiteLLM ships - Exa's entry with tiers only (what older releases bill), so a flat price there is always someone's override. + provider's mode parameter (Tavily `search_depth`, Linkup `depth`, Exa `type`). A flat price beats the tiers on + the same entry, since LiteLLM never ships both, so the flat one is someone's override. An Exa request is billed + the `costDollars.total` Exa reported for it unless its entry holds a price someone set. Args: model: Model name (e.g., "exa_ai/search", "tavily/search") @@ -109,25 +105,21 @@ def search_provider_cost_per_query( ) return (input_cost, 0.0) - exa_price_set: Final = custom_llm_provider == "exa_ai" and _has_flat_price( - _generic_search_entry(model, custom_llm_provider) - ) + pricing_model: Final = _pricing_model(model, custom_llm_provider, optional_params) reported_cost: Final = ( exa_ai_reported_cost(optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS) - if custom_llm_provider == "exa_ai" and not exa_price_set + if custom_llm_provider == "exa_ai" + and not exa_ai_price_set(_live_entry(_generic_search_entry(pricing_model, custom_llm_provider))) else None ) if reported_cost is not None: return (reported_cost, 0.0) - model_info: Final = get_model_info( - model=_pricing_model(model, custom_llm_provider, optional_params), - custom_llm_provider=custom_llm_provider, - ) + model_info: Final = get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider) # Check for tiered pricing (e.g., Exa AI based on max_results) tiered_pricing: Final = model_info.get("tiered_pricing") - if tiered_pricing and isinstance(tiered_pricing, list) and not exa_price_set: + if tiered_pricing and isinstance(tiered_pricing, list) and model_info.get("input_cost_per_query") is None: max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results cost_per_query = 0.0 diff --git a/tests/unit/llms/exa_ai/search/test_cost_calculator.py b/tests/unit/llms/exa_ai/search/test_cost_calculator.py new file mode 100644 index 00000000000..738227a3599 --- /dev/null +++ b/tests/unit/llms/exa_ai/search/test_cost_calculator.py @@ -0,0 +1,40 @@ +import json +from pathlib import Path +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_LEGACY_TIERS, ExaAITier + +REPO_ROOT: Final = Path(__file__).resolve().parents[5] +SHIPPED_COST_MAPS: Final = ( + "model_prices_and_context_window.json", + "litellm/model_prices_and_context_window_backup.json", +) + + +def _search_entries(path: str) -> dict[str, dict[str, object]]: + return { + key: entry + for key, entry in json.loads((REPO_ROOT / path).read_text()).items() + if isinstance(entry, dict) and entry.get("mode") == "search" + } + + +@pytest.mark.parametrize("path", SHIPPED_COST_MAPS) +def test_litellm_ships_exa_search_with_only_the_legacy_tiers_the_cost_calculator_treats_as_its_own(path: str) -> None: + exa_entry: Final = _search_entries(path)["exa_ai/search"] + + assert TypeAdapter(tuple[ExaAITier, ...]).validate_python(exa_entry["tiered_pricing"]) == EXA_AI_LEGACY_TIERS + assert exa_entry.get("input_cost_per_query") is None + assert [key for key in _search_entries(path) if key.startswith("exa_ai/search-")] == [] + + +@pytest.mark.parametrize("path", SHIPPED_COST_MAPS) +def test_litellm_ships_no_search_entry_with_both_a_flat_price_and_tiers(path: str) -> None: + assert [ + key + for key, entry in _search_entries(path).items() + if entry.get("input_cost_per_query") is not None and entry.get("tiered_pricing") is not None + ] == [] diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 6a450e9efaa..7918efd9575 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -14,7 +14,7 @@ import litellm from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult -from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM, exa_ai_reported_cost +from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.search.cost_calculator import search_provider_cost_per_query @@ -50,7 +50,7 @@ def test_transform_search_response_snippet_falls_back_through_content_modes( json={"results": [{"title": "Title", "url": "https://example.com", **content_fields}]}, ) - response: Final = ExaAISearchConfig().transform_search_response(raw_response, logging_obj=Mock(optional_params={})) + response: Final = ExaAISearchConfig().transform_search_response(raw_response, logging_obj=Mock()) assert response.results[0].snippet == expected_snippet @@ -102,9 +102,7 @@ def test_search_sends_max_results_as_num_results(respx_mock: respx.MockRouter) - def _transform(payload: object) -> SearchResponse: - return ExaAISearchConfig().transform_search_response( - httpx.Response(200, json=payload), logging_obj=Mock(optional_params={}) - ) + return ExaAISearchConfig().transform_search_response(httpx.Response(200, json=payload), logging_obj=Mock()) def test_transform_search_response_maps_every_exa_result_in_provider_order(): @@ -256,7 +254,9 @@ def test_transform_search_response_joins_whatever_the_highlights_value_iterates_ assert response.results[0].snippet == expected_snippet -@pytest.mark.parametrize("payload", [{}, {"results": []}, {"results": ""}, {"results": {}}, {"requestId": "abc"}]) +@pytest.mark.parametrize( + "payload", [{}, {"results": []}, {"results": None}, {"results": ""}, {"results": {}}, {"requestId": "abc"}] +) def test_transform_search_response_without_results_is_empty(payload: dict[str, object]): assert _transform(payload).results == [] @@ -384,17 +384,21 @@ def test_a_caller_cannot_set_the_cost_exa_reports(respx_mock: respx.MockRouter) assert response.hidden_params["response_cost"] == pytest.approx(_catalog_price(25)) -def test_a_reply_without_a_cost_does_not_keep_an_earlier_replys_cost() -> None: - logging_obj: Final = Mock(optional_params={}) - config: Final = ExaAISearchConfig() +@pytest.mark.usefixtures("exa_api_key") +def test_costing_an_exa_reply_directly_returns_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}) + response: Final = litellm.search(query="asyncio TaskGroup", search_provider="exa_ai", max_results=25) + assert _catalog_price(25) != EXA_REPORTED_COST - config.transform_search_response( - httpx.Response(200, json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}), logging_obj=logging_obj + completion_cost: Final = litellm.completion_cost( + completion_response=response, model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search" + ) + per_token: Final = litellm.cost_per_token( + model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search", response=response ) - assert exa_ai_reported_cost(logging_obj.optional_params) == EXA_REPORTED_COST - config.transform_search_response(httpx.Response(200, json={"results": []}), logging_obj=logging_obj) - assert exa_ai_reported_cost(logging_obj.optional_params) is None + assert completion_cost == pytest.approx(EXA_REPORTED_COST) + assert per_token == pytest.approx((EXA_REPORTED_COST, 0.0)) class _SpendCapture(CustomLogger): diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index 3428c3097e4..1eb3c7305b8 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -172,24 +172,45 @@ def test_unpriced_fallback_search_tool_bills_the_cost_exa_reports_not_the_failed assert fallback.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) +def _single_tier(price: float) -> list[dict[str, object]]: + return [{"input_cost_per_query": price, "max_results_range": [0, 100]}] + + def _register_exa_price(monkeypatch: pytest.MonkeyPatch, price: float) -> None: litellm.register_model( {"exa_ai/search": {"input_cost_per_query": price, "litellm_provider": "exa_ai", "mode": "search"}} ) +def _register_exa_tiers(monkeypatch: pytest.MonkeyPatch, price: float) -> None: + litellm.register_model( + {"exa_ai/search": {"tiered_pricing": _single_tier(price), "litellm_provider": "exa_ai", "mode": "search"}} + ) + + def _serve_exa_price_in_cost_map(monkeypatch: pytest.MonkeyPatch, price: float) -> None: monkeypatch.setitem( litellm.model_cost, "exa_ai/search", {**litellm.model_cost["exa_ai/search"], "input_cost_per_query": price} ) +def _serve_exa_tiers_in_cost_map(monkeypatch: pytest.MonkeyPatch, price: float) -> None: + monkeypatch.setitem( + litellm.model_cost, + "exa_ai/search", + {**litellm.model_cost["exa_ai/search"], "tiered_pricing": _single_tier(price)}, + ) + + @pytest.mark.usefixtures("_local_model_cost_map") -@pytest.mark.parametrize("set_exa_price", [_register_exa_price, _serve_exa_price_in_cost_map]) +@pytest.mark.parametrize( + "set_exa_price", + [_register_exa_price, _register_exa_tiers, _serve_exa_price_in_cost_map, _serve_exa_tiers_in_cost_map], +) @pytest.mark.parametrize( "exa_reply", [EXA_REPLY_WITH_COST, {"results": []}], ids=["exa-reports-a-cost", "exa-reports-no-cost"] ) -def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_the_tiers( +def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_litellms_tiers( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, set_exa_price: Callable[[pytest.MonkeyPatch, float], None], @@ -210,6 +231,72 @@ def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_the_tiers( assert response.hidden_params["response_cost"] == pytest.approx(set_price) +@pytest.mark.usefixtures("_local_model_cost_map") +def test_registering_litellms_own_exa_tiers_still_overrides_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + tier_price, _ = search_provider_cost_per_query( + model="exa_ai/search", custom_llm_provider="exa_ai", optional_params={"max_results": 25} + ) + assert tier_price != EXA_REPORTED_COST + litellm.register_model( + { + "exa_ai/search": { + "tiered_pricing": litellm.model_cost["exa_ai/search"]["tiered_pricing"], + "litellm_provider": "exa_ai", + "mode": "search", + } + } + ) + + response: Final = litellm.search( + query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key", max_results=25 + ) + + assert response.hidden_params["response_cost"] == pytest.approx(tier_price) + + +@pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize(("search_type", "expected_cost"), [("lite", 0.002), ("auto", EXA_REPORTED_COST)]) +def test_a_price_set_for_one_exa_search_type_applies_only_to_that_type( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, search_type: str, expected_cost: float +) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + monkeypatch.setitem( + litellm.model_cost, + "exa_ai/search-lite", + {"input_cost_per_query": 0.002, "litellm_provider": "exa_ai", "mode": "search"}, + ) + + response: Final = litellm.search( + query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key", type=search_type + ) + + assert json.loads(respx_mock.calls.last.request.content)["type"] == search_type + assert response.hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.usefixtures("_local_model_cost_map") +def test_a_flat_price_registered_on_a_tiered_search_entry_beats_its_tiers() -> None: + tier_price, _ = search_provider_cost_per_query( + model="firecrawl/search", custom_llm_provider="firecrawl", optional_params={"max_results": 15} + ) + registered_price: Final = 0.002 + assert tier_price != registered_price + litellm.register_model( + { + "firecrawl/search": { + "input_cost_per_query": registered_price, + "litellm_provider": "firecrawl", + "mode": "search", + } + } + ) + + assert search_provider_cost_per_query( + model="firecrawl/search", custom_llm_provider="firecrawl", optional_params={"max_results": 15} + ) == pytest.approx((registered_price, 0.0)) + + @pytest.mark.usefixtures("_local_model_cost_map") def test_the_cost_exa_reports_is_billed_when_the_cost_map_has_no_exa_entry( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch From f7c34a3331f1f0168d358de67e1d2b6f9c94d318 Mon Sep 17 00:00:00 2001 From: kesku Date: Wed, 28 Oct 2026 22:13:22 -0700 Subject: [PATCH 4/8] feat(exa): adopt non-streaming search API refresh Adapted the request contract, immutable request mapping, and HTTP wire tests from #46525, preserving the provider_metadata envelope used by this branch. Source commits: 924d0ea91908e751b9432acd07199b4beddb8bf3, d798ae5d539e5ab872cdcc2eeba89e30bcd4ae78, f691d27d5fc85619111132a40ecb4371565f7c64 --- litellm/llms/exa_ai/search/transformation.py | 92 +++++----------- litellm/llms/exa_ai/search/types.py | 99 +++++++++++++++++ tests/integration/sdk/test_exa_search_wire.py | 103 ++++++++++++++++++ 3 files changed, 229 insertions(+), 65 deletions(-) create mode 100644 litellm/llms/exa_ai/search/types.py create mode 100644 tests/integration/sdk/test_exa_search_wire.py diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index f4969bc1536..47abdb53e10 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -4,8 +4,8 @@ Calls Exa AI's /search endpoint to search the web. Exa AI API Reference: https://docs.exa.ai/reference/search """ -from collections.abc import Iterable, Mapping -from typing import Final, TypedDict +from collections.abc import Iterable, Mapping, Sequence +from typing import Final import httpx from pydantic import ConfigDict, TypeAdapter @@ -19,43 +19,17 @@ from litellm.llms.base_llm.search.transformation import ( SearchResult, ) from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM +from litellm.llms.exa_ai.search.types import ExaAISearchRequest from litellm.secret_managers.main import get_secret_str +__all__ = ["ExaAISearchConfig", "ExaAISearchRequest"] + _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True)) _STRS: Final = TypeAdapter(Iterable[str], config=ConfigDict(hide_input_in_errors=True)) _RENAMED_RESULT_FIELDS: Final = frozenset({"text", "publishedDate", "highlightScores"}) -class _ExaAISearchRequestRequired(TypedDict): - """Required fields for Exa AI Search API request.""" - - query: str # Required - search query - - -class ExaAISearchRequest(_ExaAISearchRequestRequired, total=False): - """ - Exa AI Search API request format. - Based on: https://docs.exa.ai/reference/search - """ - - type: str # Optional - search type ('keyword', 'neural', 'fast', 'auto'), default 'auto' - category: str # Optional - data category ('company', 'research paper', 'news', 'pdf', 'github', 'tweet', 'personal site', 'linkedin profile', 'financial report') - userLocation: str # Optional - two-letter ISO country code - numResults: int # Optional - number of results (max 100), default 10 - includeDomains: list[str] # Optional - list of domains to include - excludeDomains: list[str] # Optional - list of domains to exclude - startCrawlDate: str # Optional - crawl date filter (ISO 8601 format) - endCrawlDate: str # Optional - crawl date filter (ISO 8601 format) - startPublishedDate: str # Optional - published date filter (ISO 8601 format) - endPublishedDate: str # Optional - published date filter (ISO 8601 format) - includeText: list[str] # Optional - strings that must be present in webpage text - excludeText: list[str] # Optional - strings that must not be present in webpage text - context: bool | dict # Optional - format results for LLMs - moderation: bool # Optional - enable content moderation, default false - contents: dict # Optional - content retrieval options - - class ExaAISearchConfig(BaseSearchConfig): EXA_AI_API_BASE = "https://api.exa.ai" @@ -101,9 +75,9 @@ class ExaAISearchConfig(BaseSearchConfig): def transform_search_request( self, query: str | list[str], - optional_params: dict, - **kwargs, - ) -> dict: + optional_params: Mapping[str, object], + **kwargs: object, + ) -> dict[str, object]: """ Transform Search request to Exa AI API format. @@ -123,38 +97,26 @@ class ExaAISearchConfig(BaseSearchConfig): Returns: Dict with typed request data following ExaAISearchRequest spec """ - if isinstance(query, list): - # Exa AI only supports single string queries, join with spaces - query = " ".join(query) - - request_data: Final[ExaAISearchRequest] = { - "query": query, + mapped: Final = { + target: optional_params[source] + for source, target in ( + ("max_results", "numResults"), + ("search_domain_filter", "includeDomains"), + ("country", "userLocation"), + ) + if source in optional_params + } + provider_params: Final = { + key: value + for key, value in optional_params.items() + if key not in self.get_supported_perplexity_optional_params() and key not in mapped + } + return { + "query": " ".join(query) if isinstance(query, list) else query, + **mapped, + **provider_params, + **({"contents": {"text": True}} if "contents" not in provider_params else {}), } - - # Transform Perplexity unified spec parameters to Exa format - if "max_results" in optional_params: - request_data["numResults"] = optional_params["max_results"] - - if "search_domain_filter" in optional_params: - request_data["includeDomains"] = optional_params["search_domain_filter"] - - if "country" in optional_params: - request_data["userLocation"] = optional_params["country"] - - # Convert to dict before dynamic key assignments - result_data: Final = dict(request_data) - - # pass through all other parameters as-is - for param, value in optional_params.items(): - if param not in self.get_supported_perplexity_optional_params() and param not in result_data: - result_data[param] = value - - # By default, request text content if not explicitly specified - # Exa AI doesn't return content/text unless explicitly requested - if "contents" not in result_data: - result_data["contents"] = {"text": True} - - return result_data def transform_search_response( self, diff --git a/litellm/llms/exa_ai/search/types.py b/litellm/llms/exa_ai/search/types.py new file mode 100644 index 00000000000..3fd1379701f --- /dev/null +++ b/litellm/llms/exa_ai/search/types.py @@ -0,0 +1,99 @@ +"""Exa /search request contract, https://exa.ai/docs/reference/search.""" + +from collections.abc import Mapping, Sequence +from typing import Literal, TypeAlias + +from typing_extensions import ReadOnly, Required, TypedDict + +ExaSearchMode: TypeAlias = Literal["lite", "instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"] +ExaPageSection: TypeAlias = Literal["header", "navigation", "banner", "body", "sidebar", "footer", "metadata"] + + +class ExaTextOptions(TypedDict, total=False): + maxCharacters: ReadOnly[int | None] + includeHtmlTags: ReadOnly[bool | None] + verbosity: ReadOnly[Literal["compact", "standard", "full"] | None] + includeSections: ReadOnly[Sequence[ExaPageSection] | None] + excludeSections: ReadOnly[Sequence[ExaPageSection] | None] + + +class ExaHighlightsOptions(TypedDict, total=False): + """numSentences and highlightsPerUrl are deprecated compatibility fields.""" + + query: ReadOnly[str | None] + verbosity: ReadOnly[Literal["low", "medium", "high"] | None] + dynamic: ReadOnly[bool | None] + maxCharacters: ReadOnly[int | None] + numSentences: ReadOnly[int | None] + highlightsPerUrl: ReadOnly[int | None] + + +class ExaSummaryOptions(TypedDict, total=False): + query: ReadOnly[str | None] + schema: ReadOnly[Mapping[str, object] | None] + + +class ExaExtrasOptions(TypedDict, total=False): + links: ReadOnly[int | None] + imageLinks: ReadOnly[int | None] + richImageLinks: ReadOnly[int | None] + richLinks: ReadOnly[int | None] + codeBlocks: ReadOnly[int | None] + + +class ExaContextOptions(TypedDict, total=False): + maxCharacters: ReadOnly[int] + + +class ExaContentsOptions(TypedDict, total=False): + """context, livecrawl, and crawledBeforeDate are deprecated compatibility fields.""" + + text: ReadOnly[bool | ExaTextOptions | None] + highlights: ReadOnly[bool | ExaHighlightsOptions | None] + summary: ReadOnly[ExaSummaryOptions | None] + extras: ReadOnly[ExaExtrasOptions | None] + livecrawlTimeout: ReadOnly[int | None] + maxAgeHours: ReadOnly[int | None] + snapshotAsOf: ReadOnly[str | None] + subpages: ReadOnly[int | None] + subpageTarget: ReadOnly[str | Sequence[str] | None] + context: ReadOnly[bool | ExaContextOptions | None] + livecrawl: ReadOnly[Literal["never", "always", "fallback", "preferred"] | None] + crawledBeforeDate: ReadOnly[str | None] + + +class ExaCountryLocation(TypedDict): + country: ReadOnly[str] + + +class ExaCoordinateLocation(TypedDict, total=False): + country: ReadOnly[str] + latitude: Required[ReadOnly[float]] + longitude: Required[ReadOnly[float]] + + +class ExaAISearchRequest(TypedDict, total=False): + """Crawl-date filters, includeText, excludeText, and context are deprecated.""" + + query: Required[ReadOnly[str]] + type: ReadOnly[ExaSearchMode | None] + category: ReadOnly[str | None] + additionalQueries: ReadOnly[Sequence[str] | None] + userLocation: ReadOnly[str | ExaCountryLocation | ExaCoordinateLocation | None] + numResults: ReadOnly[int | None] + includeDomains: ReadOnly[Sequence[str] | None] + excludeDomains: ReadOnly[Sequence[str] | None] + startPublishedDate: ReadOnly[str | None] + endPublishedDate: ReadOnly[str | None] + moderation: ReadOnly[bool | None] + contents: ReadOnly[ExaContentsOptions | None] + outputSchema: ReadOnly[Mapping[str, object] | None] + objective: ReadOnly[str | None] + systemPrompt: ReadOnly[str | None] + userIdentifier: ReadOnly[str | None] + compliance: ReadOnly[Literal["hipaa"] | None] + startCrawlDate: ReadOnly[str | None] + endCrawlDate: ReadOnly[str | None] + includeText: ReadOnly[Sequence[str] | None] + excludeText: ReadOnly[Sequence[str] | None] + context: ReadOnly[bool | ExaContextOptions | None] diff --git a/tests/integration/sdk/test_exa_search_wire.py b/tests/integration/sdk/test_exa_search_wire.py new file mode 100644 index 00000000000..1a46db2ea17 --- /dev/null +++ b/tests/integration/sdk/test_exa_search_wire.py @@ -0,0 +1,103 @@ +import json +import threading +from typing import Final + +import pytest + +import litellm +from tests.integration._support.wire import Reply, Request, wire_server + +PAYLOAD: Final = { + "results": [{"title": "Source", "url": "https://example.com", "highlights": ["evidence"], "author": None}], + "output": {"content": {"answer": "verified"}, "grounding": []}, + "requestId": "wire-request", + "searchTime": 14, + "costDollars": {"total": 0.01}, +} +PARAMS: Final = { + "contents": {"highlights": {"dynamic": True, "verbosity": "low"}}, + "outputSchema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "systemPrompt": "Use evidence", + "type": "auto", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_exa_search_request_and_full_response_cross_real_http(asynchronous: bool) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/search" + assert request.headers["x-api-key"] == "test-key" + assert request.headers["exa-beta"] == "dynamic-highlights-2026-08-28" + assert json.loads(request.body) == {"query": "question", **PARAMS} + return Reply(body=json.dumps(PAYLOAD).encode()) + + with wire_server(respond) as server: + params: Final = { + "query": "question", + "search_provider": "exa_ai", + "api_base": server.url, + "api_key": "test-key", + "extra_headers": {"Exa-Beta": "dynamic-highlights-2026-08-28"}, + **PARAMS, + } + response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) + result: Final = json.loads(response.model_dump_json()) + assert result["provider_metadata"]["output"] == PAYLOAD["output"] + assert result["provider_metadata"]["requestId"] == PAYLOAD["requestId"] + assert result["provider_metadata"]["costDollars"] == PAYLOAD["costDollars"] + assert result["results"][0]["highlights"] == ["evidence"] + assert result["results"][0]["author"] is None + assert result["results"][0]["snippet"] == "evidence" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("schema", [False, True]) +@pytest.mark.parametrize("price_override", [None, 0.0, 0.123]) +async def test_search_logs_one_completed_response_and_cost( + asynchronous: bool, schema: bool, price_override: float | None +) -> None: + import asyncio + from collections.abc import Mapping + + from litellm.integrations.custom_logger import CustomLogger + + class Recorder(CustomLogger): + def __init__(self) -> None: + self.finished = threading.Event() + self.records = () + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.records = (*self.records, (kwargs, response_obj)) + self.finished.set() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.log_success_event(kwargs, response_obj, start_time, end_time) + + recorder: Final = Recorder() + + def respond(request: Request) -> Reply: + assert "input_cost_per_query" not in json.loads(request.body) + return Reply(body=json.dumps(PAYLOAD).encode()) + + with wire_server(respond) as server: + params: Final = { + "query": "question", + "search_provider": "exa_ai", + "api_base": server.url, + "api_key": "test-key", + **({"input_cost_per_query": price_override} if price_override is not None else {}), + "success_callback": [recorder.async_log_success_event if asynchronous else recorder], + **{key: value for key, value in PARAMS.items() if schema or key != "outputSchema"}, + } + response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) + assert await asyncio.to_thread(recorder.finished.wait, 5) + assert len(recorder.records) == 1 + details, logged = recorder.records[0] + assert logged.model_dump()["provider_metadata"]["output"] == response.model_dump()["provider_metadata"]["output"] + assert isinstance(details["standard_logging_object"], Mapping) + expected_cost: Final = PAYLOAD["costDollars"]["total"] if price_override is None else price_override + assert details["response_cost"] == expected_cost + assert details["standard_logging_object"]["response_cost"] == expected_cost + assert details["stream"] is False From e9727d2738a03e27b03da2db12755bbd6e450f8c Mon Sep 17 00:00:00 2001 From: Aaron Rodrigues Date: Wed, 28 Oct 2026 22:13:56 -0700 Subject: [PATCH 5/8] feat(exa): reuse provider list-price fallback Adapted the fallback estimator and Exa catalog entries from #46525. Follow-up refinements by Kes and Mateo retain native count precedence, summary charges, Lite pricing, and compatibility tiers. Source commits: d0a37501fc337d2ea8381157873ee60f15cb376c, d150fb91db109f0a6e9571db87bf204f00302102, 8185114b5bcd9323c871a6075fe9c743040d7e18, 78acd12678af67b0cf2967f9aa1492e8ae6bc632, f691d27d5fc85619111132a40ecb4371565f7c64, 43c59c2cd713bc6221d2df375a4997cdfb60d54f, 96a2b14005e7032734b7453eaa228663143a2fe2 Co-authored-by: kesku Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/exa_ai/search/cost_calculator.py | 46 +++++++++++++++++++ ...odel_prices_and_context_window_backup.json | 44 ++++++++++++++++++ model_prices_and_context_window.json | 44 ++++++++++++++++++ 3 files changed, 134 insertions(+) diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 5d6e38d5bad..2207ccecbb0 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -59,3 +59,49 @@ def exa_ai_price_set(entry: object) -> bool: fields: Final = _ENTRY.validate_python(entry) tiers: Final = fields.get("tiered_pricing") return fields.get("input_cost_per_query") is not None or (tiers is not None and not _legacy_tiers(tiers)) + + +EXA_AI_INCLUDED_RESULTS: Final = 10 +EXA_AI_ADDITIONAL_RESULT_COST: Final = 0.001 +EXA_AI_SUMMARY_COST_PER_PAGE: Final = 0.001 +EXA_AI_TYPES_WITHOUT_SUMMARIES: Final = frozenset({"lite"}) +_CONTENTS: Final = TypeAdapter(Mapping[str, object]) + + +def _non_negative_int(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _requested_results(optional_params: Mapping[str, object]) -> int: + for key in ("max_results", "numResults"): + requested: Final = _non_negative_int(optional_params.get(key)) + if requested is not None: + return requested + return EXA_AI_INCLUDED_RESULTS + + +def _requests_summaries(optional_params: Mapping[str, object]) -> bool: + if optional_params.get("type") in EXA_AI_TYPES_WITHOUT_SUMMARIES: + return False + contents: Final = optional_params.get("contents") + return isinstance(contents, Mapping) and _CONTENTS.validate_python(contents).get("summary") not in (None, False) + + +def exa_ai_list_price(base_price: float, optional_params: Mapping[str, object]) -> float: + """ + List price of one Exa /search request, for responses that carry no `costDollars.total`. + + Multi-query searches are joined into a single request, so this prices one request: + the search type's base price, a per-result charge above the results that price includes, + and a per-page charge for each requested result when summaries are requested on a type that offers them. + """ + results: Final = _requested_results(optional_params) + additional_results: Final = max(results - EXA_AI_INCLUDED_RESULTS, 0) + summarized_pages: Final = results if _requests_summaries(optional_params) else 0 + return ( + base_price + + additional_results * EXA_AI_ADDITIONAL_RESULT_COST + + summarized_pages * EXA_AI_SUMMARY_COST_PER_PAGE + ) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index dc043dc4d26..e000b3543bc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24315,7 +24315,11 @@ "output_vector_size": 2560 }, "exa_ai/search": { + "input_cost_per_query": 0.007, "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request, covering up to 10 results. Each result above 10 adds $0.001, and each summarized page adds $0.001. Exa's reported costDollars.total takes precedence when present, unless an operator priced this entry. tiered_pricing keeps what older releases bill for this entry. Source: https://exa.ai/pricing (2026-10-15)" + }, "mode": "search", "tiered_pricing": [ { @@ -24334,6 +24338,46 @@ } ] }, + "exa_ai/search-lite": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-22)" + }, + "mode": "search" + }, + "exa_ai/search-instant": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type instant, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep-lite": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-lite, covering up to 10 results. Source: https://exa.ai/docs/reference/pricing (2026-10-27)" + }, + "mode": "search" + }, + "exa_ai/search-deep-reasoning": { + "input_cost_per_query": 0.015, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-reasoning, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, "firecrawl/search": { "litellm_provider": "firecrawl", "mode": "search", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dc043dc4d26..e000b3543bc 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24315,7 +24315,11 @@ "output_vector_size": 2560 }, "exa_ai/search": { + "input_cost_per_query": 0.007, "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request, covering up to 10 results. Each result above 10 adds $0.001, and each summarized page adds $0.001. Exa's reported costDollars.total takes precedence when present, unless an operator priced this entry. tiered_pricing keeps what older releases bill for this entry. Source: https://exa.ai/pricing (2026-10-15)" + }, "mode": "search", "tiered_pricing": [ { @@ -24334,6 +24338,46 @@ } ] }, + "exa_ai/search-lite": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-22)" + }, + "mode": "search" + }, + "exa_ai/search-instant": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type instant, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep-lite": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-lite, covering up to 10 results. Source: https://exa.ai/docs/reference/pricing (2026-10-27)" + }, + "mode": "search" + }, + "exa_ai/search-deep-reasoning": { + "input_cost_per_query": 0.015, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-reasoning, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, "firecrawl/search": { "litellm_provider": "firecrawl", "mode": "search", From ebee797b7181a003ae4810c6b5ad479eb8762ad4 Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 22:20:44 -0700 Subject: [PATCH 6/8] fix(exa): unify receipt and operator pricing precedence Keep receipts usable after serialization and without catalog entries. Preserve request modes for direct costing, apply generic custom-map prices across modes, and use the same result count for native and unified requests. The latest Exa price registration replaces the previous price kind. Adapted the public receipt lookup and canonical catalog URL from 96a2b14005e7032734b7453eaa228663143a2fe2 and ffe8a669b0021bde84060b2fa62aca91b1a7878c. Keep pricing detection local to Exa and compare catalog entries individually rather than fingerprinting the entire installed catalog. Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 3 +- litellm/constants.py | 3 + litellm/cost_calculator.py | 2 + litellm/llms/exa_ai/search/cost_calculator.py | 63 +++++++-- litellm/llms/exa_ai/search/transformation.py | 15 +- litellm/search/cost_calculator.py | 54 +++++-- litellm/utils.py | 8 +- tests/integration/sdk/test_exa_search_wire.py | 4 +- .../exa_ai/search/test_cost_calculator.py | 53 +++++-- .../llms/exa_ai/search/test_transformation.py | 2 +- tests/unit/search/test_main.py | 133 ++++++++++++++++-- tests/unit/test_utils.py | 14 ++ 12 files changed, 298 insertions(+), 56 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 8dfe5adca1c..e8ed4a9a832 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -41,6 +41,7 @@ from litellm._logging import ( ) import re from litellm.constants import ( + DEFAULT_MODEL_COST_MAP_URL, DEFAULT_BATCH_SIZE, DEFAULT_FLUSH_INTERVAL_SECONDS, ROUTER_MAX_FALLBACKS, @@ -410,7 +411,7 @@ aclient_session: Optional[Union[httpx.AsyncClient, DefaultAsyncHttpxClient]] = N model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks' model_cost_map_url: str = os.getenv( "LITELLM_MODEL_COST_MAP_URL", - "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json", + DEFAULT_MODEL_COST_MAP_URL, ) blog_posts_url: str = os.getenv( "LITELLM_BLOG_POSTS_URL", diff --git a/litellm/constants.py b/litellm/constants.py index 1d5c6404bfd..9257a86dffa 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1738,6 +1738,9 @@ ALLOWED_VERTEX_AI_PASSTHROUGH_HEADERS: Final = { PASS_THROUGH_HEADER_PREFIX: Final = "x-pass-" TYPESAFE_DANGER_ZONE_HEADER_PREFIX: Final = "typesafe-danger-zone-" INTERNAL_KWARG_PREFIX: Final = "_litellm_" +DEFAULT_MODEL_COST_MAP_URL: Final = ( + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json" +) CONTROL_OPTIONS_KEY: Final = f"{INTERNAL_KWARG_PREFIX}control" SESSION_ID_FROM_TRACE_ID_PARAM_KEY: Final = f"{INTERNAL_KWARG_PREFIX}session_id_from_trace_id" diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4c46ab34468..17f626da070 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -709,6 +709,7 @@ def cost_per_token( custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), + response=response, ) elif ( per_second_cost := _per_second_pricing_cost( @@ -1913,6 +1914,7 @@ def completion_cost( model=search_model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries, + response=completion_response, optional_params={ **(get_hidden_params(completion_response) or {}), **(optional_params or {}), diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 2207ccecbb0..5b8d6f96602 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -1,14 +1,30 @@ from collections.abc import Mapping +from types import MappingProxyType from typing import Annotated, Final from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError -from litellm.constants import INTERNAL_KWARG_PREFIX -from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry +from litellm.constants import DEFAULT_MODEL_COST_MAP_URL, INTERNAL_KWARG_PREFIX +from litellm.litellm_core_utils.get_model_cost_map import ( + AliasedCostEntry, + GetModelCostMap, + get_model_cost_map_source_info, +) +from litellm.llms.base_llm.search.transformation import SearchResponse EXA_AI_COST_DOLLARS_PARAM: Final = f"{INTERNAL_KWARG_PREFIX}exa_ai_cost_dollars" _PRICE_FIELDS: Final = frozenset({"input_cost_per_query", "tiered_pricing"}) _ENTRY: Final = TypeAdapter(Mapping[str, object]) +EXA_AI_CATALOG_PRICES: Final[Mapping[str, float]] = MappingProxyType( + { + "exa_ai/search": 0.007, + "exa_ai/search-lite": 0.004, + "exa_ai/search-instant": 0.004, + "exa_ai/search-deep": 0.012, + "exa_ai/search-deep-lite": 0.012, + "exa_ai/search-deep-reasoning": 0.015, + } +) class _ExaCostDollars(BaseModel): @@ -33,10 +49,15 @@ EXA_AI_LEGACY_TIERS: Final = ( """The tiers LiteLLM ships on `exa_ai/search`. Only older releases bill them, so they never change.""" -def exa_ai_reported_cost(params: Mapping[str, object]) -> float | None: +def exa_ai_reported_cost(params: Mapping[str, object], response: object = None) -> float | None: """The `costDollars.total` Exa reported for the request, when it is a finite non-negative number.""" try: - return _ExaCostDollars.model_validate(params.get(EXA_AI_COST_DOLLARS_PARAM)).total + payload: Final = response.model_extra if isinstance(response, SearchResponse) else response + metadata: Final = _ENTRY.validate_python(payload or {}).get("provider_metadata") + receipt: Final = _ENTRY.validate_python(metadata or {}).get("costDollars") + return _ExaCostDollars.model_validate( + receipt if receipt is not None else params.get(EXA_AI_COST_DOLLARS_PARAM) + ).total except ValidationError: return None @@ -48,17 +69,29 @@ def _legacy_tiers(tiers: object) -> bool: return False -def exa_ai_price_set(entry: object) -> bool: - """Whether an Exa price entry holds a price someone set, which replaces the cost Exa reports. LiteLLM ships - Exa with its legacy tiers only, so anything else there was set by `register_model()`, an admin's own cost map, - or an edited bundled file.""" - if isinstance(entry, AliasedCostEntry) and entry.registered_fields & _PRICE_FIELDS: - return True +def exa_ai_price_fields(model: str, entry: object) -> frozenset[str]: if not isinstance(entry, Mapping): - return False + return frozenset() fields: Final = _ENTRY.validate_python(entry) - tiers: Final = fields.get("tiered_pricing") - return fields.get("input_cost_per_query") is not None or (tiers is not None and not _legacy_tiers(tiers)) + registered: Final = entry.registered_fields & _PRICE_FIELDS if isinstance(entry, AliasedCostEntry) else frozenset() + if registered: + return registered + source: Final = get_model_cost_map_source_info() + if source["source"] == "remote" and source["url"] == DEFAULT_MODEL_COST_MAP_URL: + loaded: Final = GetModelCostMap.loaded_model_cost_map().get(model, {}) + return frozenset( + name for name in _PRICE_FIELDS if fields.get(name) is not None and fields[name] != loaded.get(name) + ) + return frozenset( + name + for name in _PRICE_FIELDS + if fields.get(name) is not None + and ( + fields[name] != EXA_AI_CATALOG_PRICES.get(model) + if name == "input_cost_per_query" + else not _legacy_tiers(fields[name]) + ) + ) EXA_AI_INCLUDED_RESULTS: Final = 10 @@ -74,7 +107,7 @@ def _non_negative_int(value: object) -> int | None: return value -def _requested_results(optional_params: Mapping[str, object]) -> int: +def exa_ai_requested_results(optional_params: Mapping[str, object]) -> int: for key in ("max_results", "numResults"): requested: Final = _non_negative_int(optional_params.get(key)) if requested is not None: @@ -97,7 +130,7 @@ def exa_ai_list_price(base_price: float, optional_params: Mapping[str, object]) the search type's base price, a per-result charge above the results that price includes, and a per-page charge for each requested result when summaries are requested on a type that offers them. """ - results: Final = _requested_results(optional_params) + results: Final = exa_ai_requested_results(optional_params) additional_results: Final = max(results - EXA_AI_INCLUDED_RESULTS, 0) summarized_pages: Final = results if _requests_summaries(optional_params) else 0 return ( diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 47abdb53e10..b8b76404bcc 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -122,7 +122,7 @@ class ExaAISearchConfig(BaseSearchConfig): self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, - **kwargs, + **kwargs: object, ) -> SearchResponse: """ Transform Exa AI API response to LiteLLM unified SearchResponse format. @@ -159,7 +159,18 @@ class ExaAISearchConfig(BaseSearchConfig): "provider_metadata": {key: value for key, value in response_json.items() if key != "results"}, } ) - set_hidden_params(response, {EXA_AI_COST_DOLLARS_PARAM: response_json.get("costDollars")}) + params: Final = _JSON_OBJECT.validate_python(kwargs.get("optional_params") or {}) + set_hidden_params( + response, + { + **{ + key: value + for key, value in params.items() + if key in ("type", "max_results", "numResults", "contents") + }, + EXA_AI_COST_DOLLARS_PARAM: response_json.get("costDollars"), + }, + ) return response diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 5eedd74d783..b5efe0a68c9 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -10,7 +10,12 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap -from litellm.llms.exa_ai.search.cost_calculator import exa_ai_price_set, exa_ai_reported_cost +from litellm.llms.exa_ai.search.cost_calculator import ( + exa_ai_list_price, + exa_ai_price_fields, + exa_ai_reported_cost, + exa_ai_requested_results, +) from litellm.utils import get_model_info PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( @@ -49,6 +54,12 @@ def _registered_at_runtime(entry: str) -> bool: return live != GetModelCostMap.loaded_model_cost_map().get(entry) # pyright: ignore[reportUnknownVariableType] # `live` comes from the untyped registry +def _operator_priced(model: str, provider: str | None) -> bool: + if provider == "exa_ai": + return bool(exa_ai_price_fields(model, _live_entry(model))) + return _registered_at_runtime(model) + + def _pricing_model( model: str, custom_llm_provider: str | None, @@ -61,10 +72,14 @@ def _pricing_model( generic_model: Final = f"{custom_llm_provider}/search" if mode_param is None or model not in (generic_model, f"{custom_llm_provider}/{generic_model}"): return model - mode: Final = (optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS).get(mode_param) + mode: Final = (optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS).get( + mode_param, "auto" if custom_llm_provider == "exa_ai" else None + ) mode_model: Final = f"{custom_llm_provider}/search-{mode}" priced: Final = mode_model in litellm.model_cost # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed - if not priced or (_registered_at_runtime(generic_model) and not _registered_at_runtime(mode_model)): + if not priced or ( + _operator_priced(generic_model, custom_llm_provider) and not _operator_priced(mode_model, custom_llm_provider) + ): return model return mode_model @@ -74,15 +89,15 @@ def search_provider_cost_per_query( custom_llm_provider: str | None = None, number_of_queries: int = 1, optional_params: Mapping[str, object] | None = None, + response: object = None, ) -> tuple[float, float]: """ Calculate cost for search-only providers. Returns (input_cost, output_cost) where input_cost = queries * cost_per_query Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the - provider's mode parameter (Tavily `search_depth`, Linkup `depth`, Exa `type`). A flat price beats the tiers on - the same entry, since LiteLLM never ships both, so the flat one is someone's override. An Exa request is billed - the `costDollars.total` Exa reported for it unless its entry holds a price someone set. + provider's mode parameter (Tavily `search_depth`, Linkup `depth`, Exa `type`). An Exa request uses an operator's + price, otherwise its reported total, otherwise estimated list price. Args: model: Model name (e.g., "exa_ai/search", "tavily/search") @@ -106,21 +121,36 @@ def search_provider_cost_per_query( return (input_cost, 0.0) pricing_model: Final = _pricing_model(model, custom_llm_provider, optional_params) - reported_cost: Final = ( - exa_ai_reported_cost(optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS) + params: Final = optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS + exa_price_fields: Final = ( + exa_ai_price_fields( + _generic_search_entry(pricing_model, custom_llm_provider), + _live_entry(_generic_search_entry(pricing_model, custom_llm_provider)), + ) if custom_llm_provider == "exa_ai" - and not exa_ai_price_set(_live_entry(_generic_search_entry(pricing_model, custom_llm_provider))) - else None + else frozenset() + ) + reported_cost: Final = ( + exa_ai_reported_cost(params, response) if custom_llm_provider == "exa_ai" and not exa_price_fields else None ) if reported_cost is not None: return (reported_cost, 0.0) model_info: Final = get_model_info(model=pricing_model, custom_llm_provider=custom_llm_provider) + if custom_llm_provider == "exa_ai" and not exa_price_fields: + return (exa_ai_list_price(float(model_info.get("input_cost_per_query") or 0.0), params), 0.0) # Check for tiered pricing (e.g., Exa AI based on max_results) tiered_pricing: Final = model_info.get("tiered_pricing") - if tiered_pricing and isinstance(tiered_pricing, list) and model_info.get("input_cost_per_query") is None: - max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results + use_tiers: Final = ( + "tiered_pricing" in exa_price_fields and "input_cost_per_query" not in exa_price_fields + if custom_llm_provider == "exa_ai" + else model_info.get("input_cost_per_query") is None + ) + if tiered_pricing and isinstance(tiered_pricing, list) and use_tiers: + max_results: Final = ( + exa_ai_requested_results(params) if custom_llm_provider == "exa_ai" else params.get("max_results", 10) + ) cost_per_query = 0.0 for tier in tiered_pricing: diff --git a/litellm/utils.py b/litellm/utils.py index 1ef39059e1b..0d3d1d19173 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3515,9 +3515,13 @@ def _replace_cost_map_entry( previous_fields: Final[frozenset[str]] = ( current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset() ) - registered_fields: Final = previous_fields | frozenset( - name for name, value in registration.items() if value is not None + supplied_fields: Final = frozenset(name for name, value in registration.items() if value is not None) + replaced_fields: Final = ( + frozenset({"input_cost_per_query", "tiered_pricing"}) + if merged.get("litellm_provider") == "exa_ai" and supplied_fields & {"input_cost_per_query", "tiered_pricing"} + else frozenset() ) + registered_fields: Final = (previous_fields - replaced_fields) | supplied_fields sharing_names: Final = tuple(name for name in names if cost_map.get(name) is current) cost_map.update(dict.fromkeys((key, *sharing_names), AliasedCostEntry(merged, names, registered_fields))) diff --git a/tests/integration/sdk/test_exa_search_wire.py b/tests/integration/sdk/test_exa_search_wire.py index 1a46db2ea17..c27f0cfc548 100644 --- a/tests/integration/sdk/test_exa_search_wire.py +++ b/tests/integration/sdk/test_exa_search_wire.py @@ -95,7 +95,9 @@ async def test_search_logs_one_completed_response_and_cost( assert await asyncio.to_thread(recorder.finished.wait, 5) assert len(recorder.records) == 1 details, logged = recorder.records[0] - assert logged.model_dump()["provider_metadata"]["output"] == response.model_dump()["provider_metadata"]["output"] + assert ( + logged.model_dump()["provider_metadata"]["output"] == response.model_dump()["provider_metadata"]["output"] + ) assert isinstance(details["standard_logging_object"], Mapping) expected_cost: Final = PAYLOAD["costDollars"]["total"] if price_override is None else price_override assert details["response_cost"] == expected_cost diff --git a/tests/unit/llms/exa_ai/search/test_cost_calculator.py b/tests/unit/llms/exa_ai/search/test_cost_calculator.py index 738227a3599..a0093206a0d 100644 --- a/tests/unit/llms/exa_ai/search/test_cost_calculator.py +++ b/tests/unit/llms/exa_ai/search/test_cost_calculator.py @@ -1,11 +1,20 @@ import json +from collections.abc import Mapping from pathlib import Path from typing import Final import pytest from pydantic import TypeAdapter -from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_LEGACY_TIERS, ExaAITier +from litellm.llms.exa_ai.search.cost_calculator import ( + EXA_AI_ADDITIONAL_RESULT_COST, + EXA_AI_CATALOG_PRICES, + EXA_AI_INCLUDED_RESULTS, + EXA_AI_LEGACY_TIERS, + EXA_AI_SUMMARY_COST_PER_PAGE, + ExaAITier, + exa_ai_list_price, +) REPO_ROOT: Final = Path(__file__).resolve().parents[5] SHIPPED_COST_MAPS: Final = ( @@ -23,18 +32,34 @@ def _search_entries(path: str) -> dict[str, dict[str, object]]: @pytest.mark.parametrize("path", SHIPPED_COST_MAPS) -def test_litellm_ships_exa_search_with_only_the_legacy_tiers_the_cost_calculator_treats_as_its_own(path: str) -> None: - exa_entry: Final = _search_entries(path)["exa_ai/search"] - - assert TypeAdapter(tuple[ExaAITier, ...]).validate_python(exa_entry["tiered_pricing"]) == EXA_AI_LEGACY_TIERS - assert exa_entry.get("input_cost_per_query") is None - assert [key for key in _search_entries(path) if key.startswith("exa_ai/search-")] == [] +def test_shipped_exa_prices_agree_with_the_receipt_override_baseline(path: str) -> None: + entries: Final = _search_entries(path) + assert { + key: entry["input_cost_per_query"] for key, entry in entries.items() if key.startswith("exa_ai/") + } == EXA_AI_CATALOG_PRICES + assert ( + TypeAdapter(tuple[ExaAITier, ...]).validate_python(entries["exa_ai/search"]["tiered_pricing"]) + == EXA_AI_LEGACY_TIERS + ) -@pytest.mark.parametrize("path", SHIPPED_COST_MAPS) -def test_litellm_ships_no_search_entry_with_both_a_flat_price_and_tiers(path: str) -> None: - assert [ - key - for key, entry in _search_entries(path).items() - if entry.get("input_cost_per_query") is not None and entry.get("tiered_pricing") is not None - ] == [] +@pytest.mark.parametrize( + ("params", "extra_results", "summaries"), + ( + ({}, 0, 0), + ({"max_results": None}, 0, 0), + ({"max_results": True, "numResults": EXA_AI_INCLUDED_RESULTS + 3}, 3, 0), + ({"numResults": EXA_AI_INCLUDED_RESULTS + 3}, 3, 0), + ({"max_results": EXA_AI_INCLUDED_RESULTS + 2, "numResults": 100}, 2, 0), + ({"max_results": 3, "contents": {"summary": {}}}, 0, 3), + ({"max_results": 3, "contents": {"summary": False}}, 0, 0), + ({"type": "lite", "max_results": 3, "contents": {"summary": {}}}, 0, 0), + ), +) +def test_fallback_adds_only_requested_results_and_summaries( + params: Mapping[str, object], extra_results: int, summaries: int +) -> None: + base_price: Final = 0.123 + assert exa_ai_list_price(base_price, params) == pytest.approx( + base_price + extra_results * EXA_AI_ADDITIONAL_RESULT_COST + summaries * EXA_AI_SUMMARY_COST_PER_PAGE + ) diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 7918efd9575..389941dba4d 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -19,7 +19,7 @@ from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" -EXA_REPORTED_COST: Final = 0.022 +EXA_REPORTED_COST: Final = 0.123456 _LOGGED: Final = TypeAdapter(Mapping[str, object]) diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index 1eb3c7305b8..5ae3b25fcf8 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -1,6 +1,7 @@ import datetime import json -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass from typing import Final import httpx @@ -8,6 +9,7 @@ import pytest import respx import litellm +from litellm.constants import DEFAULT_MODEL_COST_MAP_URL from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.litellm_logging import Logging from litellm.search.cost_calculator import search_provider_cost_per_query @@ -15,7 +17,7 @@ from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" LINKUP_SEARCH_URL: Final = "https://api.linkup.so/v1/search" -EXA_REPORTED_COST: Final = 0.022 +EXA_REPORTED_COST: Final = 0.123456 EXA_REPLY_WITH_COST: Final = { "results": [{"title": "t", "url": "https://example.com"}], "costDollars": {"total": EXA_REPORTED_COST}, @@ -234,9 +236,7 @@ def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_litellms_ti @pytest.mark.usefixtures("_local_model_cost_map") def test_registering_litellms_own_exa_tiers_still_overrides_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) - tier_price, _ = search_provider_cost_per_query( - model="exa_ai/search", custom_llm_provider="exa_ai", optional_params={"max_results": 25} - ) + tier_price: Final = litellm.model_cost["exa_ai/search"]["tiered_pricing"][0]["input_cost_per_query"] assert tier_price != EXA_REPORTED_COST litellm.register_model( { @@ -298,13 +298,20 @@ def test_a_flat_price_registered_on_a_tiered_search_entry_beats_its_tiers() -> N @pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize("mode", ("auto", "deep")) def test_the_cost_exa_reports_is_billed_when_the_cost_map_has_no_exa_entry( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, mode: str ) -> None: respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) - monkeypatch.delitem(litellm.model_cost, "exa_ai/search") + monkeypatch.setattr( + litellm, + "model_cost", + {key: value for key, value in litellm.model_cost.items() if not key.startswith("exa_ai/")}, + ) - response: Final = litellm.search(query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key") + response: Final = litellm.search( + query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key", type=mode + ) assert response.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) @@ -449,6 +456,116 @@ def test_a_runtime_price_on_the_mode_entry_wins_over_a_runtime_price_on_the_gene assert cost == pytest.approx((0.011, 0.0)) +@pytest.mark.usefixtures("_local_model_cost_map") +def test_exa_receipt_survives_public_response_serialization(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + response: Final = litellm.search(query="receipt", search_provider="exa_ai", api_key="test-key") + + assert litellm.completion_cost( + completion_response=response.model_dump(), + model="exa_ai/search", + custom_llm_provider="exa_ai", + call_type="search", + ) == pytest.approx(EXA_REPORTED_COST) + + +@pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize("params", ({"type": "deep"}, {}), ids=("explicit-mode", "default-mode")) +def test_exa_pricing_apis_keep_the_requested_mode(respx_mock: respx.MockRouter, params: Mapping[str, object]) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + mode: Final = params.get("type", "auto") + price: Final = 0.321 + litellm.register_model( + {f"exa_ai/search-{mode}": {"input_cost_per_query": price, "litellm_provider": "exa_ai", "mode": "search"}}, + persist_across_reloads=False, + ) + response: Final = litellm.search(query="mode", search_provider="exa_ai", api_key="test-key", **params) + + assert response.hidden_params["response_cost"] == pytest.approx(price) + assert sum( + litellm.cost_per_token( + model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search", response=response + ) + ) == pytest.approx(price) + assert litellm.completion_cost( + completion_response=response, model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search" + ) == pytest.approx(price) + + +@pytest.mark.usefixtures("_local_model_cost_map") +def test_exa_native_result_count_bills_the_same_operator_tier(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + price: Final = 0.456 + litellm.register_model( + { + "exa_ai/search": { + "tiered_pricing": [ + {"input_cost_per_query": 0.123, "max_results_range": [0, 10]}, + {"input_cost_per_query": price, "max_results_range": [11, 100]}, + ] + } + }, + persist_across_reloads=False, + ) + for params in ({"max_results": 50}, {"numResults": 50}, {"max_results": None, "numResults": 50}): + response: Final = litellm.search(query="count", search_provider="exa_ai", api_key="test-key", **params) + assert json.loads(route.calls.last.request.content)["numResults"] == 50 + assert response.hidden_params["response_cost"] == pytest.approx(price) + + +@dataclass(frozen=True, slots=True) +class _CostMapServer: + body: bytes + + def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: + return httpx.Response(200, content=self.body, request=httpx.Request("GET", url)) + + +@pytest.fixture +def _restore_catalog_source(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + yield + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=DEFAULT_MODEL_COST_MAP_URL)) + + +@pytest.mark.usefixtures("_restore_catalog_source") +@pytest.mark.parametrize( + ("source", "price", "expected_cost"), + ( + (DEFAULT_MODEL_COST_MAP_URL, 0.321, EXA_REPORTED_COST), + ("https://prices.invalid/map.json", 0.321, 0.321), + ("https://prices.invalid/map.json", None, EXA_REPORTED_COST), + ), + ids=("updated-official-catalog", "operator-map", "mirror"), +) +@pytest.mark.parametrize("mode", ("auto", "lite", "deep")) +def test_exa_catalog_source_preserves_receipts_and_generic_operator_prices( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + source: str, + price: float | None, + expected_cost: float, + mode: str, +) -> None: + respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) + catalog: Final = GetModelCostMap.load_local_model_cost_map() + served: Final = { + **catalog, + "exa_ai/search": { + **catalog["exa_ai/search"], + **({"input_cost_per_query": price} if price is not None else {}), + }, + } + monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) + monkeypatch.setattr( + litellm, + "model_cost", + litellm.get_model_cost_map(url=source, max_attempts=1, client=_CostMapServer(json.dumps(served).encode())), + ) + response: Final = litellm.search(query="catalog", search_provider="exa_ai", api_key="test-key", type=mode) + assert response.hidden_params["response_cost"] == pytest.approx(expected_cost) + + @pytest.mark.usefixtures("_search_mode_prices") def test_sync_tavily_advanced_search_reports_the_advanced_price(respx_mock: respx.MockRouter) -> None: respx_mock.post(TAVILY_SEARCH_URL).respond(json={"results": [{"title": "t", "url": "https://example.com"}]}) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 146e2df6413..b7bdb261d5b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -2325,6 +2325,20 @@ def test_register_model_price_is_used_for_completion_cost() -> None: assert cost == pytest.approx(0.02) +@pytest.mark.parametrize("flat_first", (True, False)) +def test_latest_exa_price_registration_replaces_the_previous_price_kind( + local_model_cost_map: None, flat_first: bool +) -> None: + flat: Final = {"input_cost_per_query": 0.321} + tiered: Final = {"tiered_pricing": [{"input_cost_per_query": 0.456, "max_results_range": [0, 100]}]} + for price in (flat, tiered) if flat_first else (tiered, flat): + litellm.register_model({"exa_ai/search": price}, persist_across_reloads=False) + + assert litellm.completion_cost( + model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search" + ) == pytest.approx(0.456 if flat_first else 0.321) + + @respx.mock @pytest.mark.usefixtures("isolated_openai_model_sets", "local_model_cost_map") def test_completion_registers_test_owned_model_prices() -> None: From a27503f67f8a027820b0fa2cdbb661cec6dc250d Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 22:27:10 -0700 Subject: [PATCH 7/8] fix(exa): satisfy typing and documentation checks --- litellm/cost_calculator.py | 2 +- litellm/llms/exa_ai/search/cost_calculator.py | 17 +++++++++++------ litellm/llms/exa_ai/search/transformation.py | 2 +- litellm/search/cost_calculator.py | 3 ++- litellm/utils.py | 2 +- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 17f626da070..04e399851e0 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1914,7 +1914,7 @@ def completion_cost( model=search_model, custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries, - response=completion_response, + response=_ATTRIBUTE_VALUE.validate_python(completion_response), optional_params={ **(get_hidden_params(completion_response) or {}), **(optional_params or {}), diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 5b8d6f96602..8477a913754 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -73,7 +73,9 @@ def exa_ai_price_fields(model: str, entry: object) -> frozenset[str]: if not isinstance(entry, Mapping): return frozenset() fields: Final = _ENTRY.validate_python(entry) - registered: Final = entry.registered_fields & _PRICE_FIELDS if isinstance(entry, AliasedCostEntry) else frozenset() + registered: Final[frozenset[str]] = ( + entry.registered_fields & _PRICE_FIELDS if isinstance(entry, AliasedCostEntry) else frozenset() + ) if registered: return registered source: Final = get_model_cost_map_source_info() @@ -108,11 +110,14 @@ def _non_negative_int(value: object) -> int | None: def exa_ai_requested_results(optional_params: Mapping[str, object]) -> int: - for key in ("max_results", "numResults"): - requested: Final = _non_negative_int(optional_params.get(key)) - if requested is not None: - return requested - return EXA_AI_INCLUDED_RESULTS + return next( + ( + requested + for key in ("max_results", "numResults") + if (requested := _non_negative_int(optional_params.get(key))) is not None + ), + EXA_AI_INCLUDED_RESULTS, + ) def _requests_summaries(optional_params: Mapping[str, object]) -> bool: diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index b8b76404bcc..5d2ff5816c8 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -4,7 +4,7 @@ Calls Exa AI's /search endpoint to search the web. Exa AI API Reference: https://docs.exa.ai/reference/search """ -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping from typing import Final import httpx diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index b5efe0a68c9..df2c527e638 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -104,6 +104,7 @@ def search_provider_cost_per_query( custom_llm_provider: Provider name (e.g., "exa_ai", "tavily") number_of_queries: Number of search queries performed (default: 1) optional_params: Optional parameters including max_results for tiered pricing + response: Search response carrying Exa's reported cost Returns: Tuple of (input_cost, output_cost) where output_cost is always 0.0 @@ -122,7 +123,7 @@ def search_provider_cost_per_query( pricing_model: Final = _pricing_model(model, custom_llm_provider, optional_params) params: Final = optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS - exa_price_fields: Final = ( + exa_price_fields: Final[frozenset[str]] = ( exa_ai_price_fields( _generic_search_entry(pricing_model, custom_llm_provider), _live_entry(_generic_search_entry(pricing_model, custom_llm_provider)), diff --git a/litellm/utils.py b/litellm/utils.py index 0d3d1d19173..78b2c8e9557 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3516,7 +3516,7 @@ def _replace_cost_map_entry( current.registered_fields if isinstance(current, AliasedCostEntry) else frozenset() ) supplied_fields: Final = frozenset(name for name, value in registration.items() if value is not None) - replaced_fields: Final = ( + replaced_fields: Final[frozenset[str]] = ( frozenset({"input_cost_per_query", "tiered_pricing"}) if merged.get("litellm_provider") == "exa_ai" and supplied_fields & {"input_cost_per_query", "tiered_pricing"} else frozenset() From 2c9c4eccd312dba2ebadd1faaf5e4268d1a1a07b Mon Sep 17 00:00:00 2001 From: shrey kharbanda Date: Wed, 28 Oct 2026 22:49:56 -0700 Subject: [PATCH 8/8] refactor(exa): reuse pricing helpers and consolidate search tests --- litellm/llms/exa_ai/search/cost_calculator.py | 3 +- litellm/search/cost_calculator.py | 23 +-- .../sdk/test_exa_search_envelope_sdk.py | 116 +++++++++---- tests/integration/sdk/test_exa_search_wire.py | 105 ------------ .../llms/exa_ai/search/test_transformation.py | 54 +----- tests/unit/search/test_main.py | 155 +++++------------- 6 files changed, 140 insertions(+), 316 deletions(-) delete mode 100644 tests/integration/sdk/test_exa_search_wire.py diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py index 8477a913754..44b72f9fb0d 100644 --- a/litellm/llms/exa_ai/search/cost_calculator.py +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -100,7 +100,6 @@ EXA_AI_INCLUDED_RESULTS: Final = 10 EXA_AI_ADDITIONAL_RESULT_COST: Final = 0.001 EXA_AI_SUMMARY_COST_PER_PAGE: Final = 0.001 EXA_AI_TYPES_WITHOUT_SUMMARIES: Final = frozenset({"lite"}) -_CONTENTS: Final = TypeAdapter(Mapping[str, object]) def _non_negative_int(value: object) -> int | None: @@ -124,7 +123,7 @@ def _requests_summaries(optional_params: Mapping[str, object]) -> bool: if optional_params.get("type") in EXA_AI_TYPES_WITHOUT_SUMMARIES: return False contents: Final = optional_params.get("contents") - return isinstance(contents, Mapping) and _CONTENTS.validate_python(contents).get("summary") not in (None, False) + return isinstance(contents, Mapping) and _ENTRY.validate_python(contents).get("summary") not in (None, False) def exa_ai_list_price(base_price: float, optional_params: Mapping[str, object]) -> float: diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index df2c527e638..870aea0347a 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -39,25 +39,14 @@ def _provider_usage( return None -def _generic_search_entry(model: str, custom_llm_provider: str | None) -> str: - generic_model: Final = f"{custom_llm_provider}/search" - return generic_model if model == f"{custom_llm_provider}/{generic_model}" else model - - def _live_entry(entry: str) -> object: return litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed -def _registered_at_runtime(entry: str) -> bool: - """Whether `entry`'s live price differs from the catalog as loaded, i.e. `register_model` changed it.""" - live: Final[object] = litellm.model_cost.get(entry) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # registry is dynamically typed - return live != GetModelCostMap.loaded_model_cost_map().get(entry) # pyright: ignore[reportUnknownVariableType] # `live` comes from the untyped registry - - def _operator_priced(model: str, provider: str | None) -> bool: if provider == "exa_ai": return bool(exa_ai_price_fields(model, _live_entry(model))) - return _registered_at_runtime(model) + return _live_entry(model) != GetModelCostMap.loaded_model_cost_map().get(model) def _pricing_model( @@ -122,14 +111,12 @@ def search_provider_cost_per_query( return (input_cost, 0.0) pricing_model: Final = _pricing_model(model, custom_llm_provider, optional_params) + exa_model: Final = ( + pricing_model.removeprefix("exa_ai/") if pricing_model == "exa_ai/exa_ai/search" else pricing_model + ) params: Final = optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS exa_price_fields: Final[frozenset[str]] = ( - exa_ai_price_fields( - _generic_search_entry(pricing_model, custom_llm_provider), - _live_entry(_generic_search_entry(pricing_model, custom_llm_provider)), - ) - if custom_llm_provider == "exa_ai" - else frozenset() + exa_ai_price_fields(exa_model, _live_entry(exa_model)) if custom_llm_provider == "exa_ai" else frozenset() ) reported_cost: Final = ( exa_ai_reported_cost(params, response) if custom_llm_provider == "exa_ai" and not exa_price_fields else None diff --git a/tests/integration/sdk/test_exa_search_envelope_sdk.py b/tests/integration/sdk/test_exa_search_envelope_sdk.py index 3ab2a1a4969..33bc0402399 100644 --- a/tests/integration/sdk/test_exa_search_envelope_sdk.py +++ b/tests/integration/sdk/test_exa_search_envelope_sdk.py @@ -1,5 +1,8 @@ +import asyncio import json -import uuid +import threading +from collections.abc import Mapping +from datetime import datetime from typing import Final import pytest @@ -7,15 +10,18 @@ from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue import litellm +from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.search.transformation import SearchResponse pytestmark: Final = pytest.mark.timeout(60) +_REPORTED_COST: Final = 0.005 _ENVELOPE: Final[dict[str, JsonValue]] = { "requestId": "exa-request-sdk", + "output": {"content": {"answer": "verified"}, "grounding": []}, "resolvedSearchType": "neural", "searchTime": 211.0, - "costDollars": {"total": 0.005, "search": {"neural": 0.005}}, + "costDollars": {"total": _REPORTED_COST, "search": {"neural": _REPORTED_COST}}, } _RESULT: Final[dict[str, JsonValue]] = { "id": "https://result.invalid/sdk", @@ -23,6 +29,7 @@ _RESULT: Final[dict[str, JsonValue]] = { "url": "https://result.invalid/sdk", "publishedDate": "2026-02-01T00:00:00.000Z", "score": 0.73, + "author": None, "highlights": ["First line.", "Second line."], "highlightScores": [0.61, 0.42], } @@ -39,6 +46,7 @@ _EXPECTED: Final[dict[str, JsonValue]] = { "highlights": ["First line.", "Second line."], "highlight_scores": [0.61, 0.42], "score": 0.73, + "author": None, } ], "provider_metadata": _ENVELOPE, @@ -50,34 +58,84 @@ def _respond(request: Request) -> Reply: return Reply(body=json.dumps({**_ENVELOPE, "results": [_RESULT]}).encode()) -def test_sync_exa_search_returns_highlights_scores_and_the_response_envelope() -> None: - with wire_server(_respond) as wire: - response: Final = litellm.search( - query=f"sdk sync {uuid.uuid4().hex}", - search_provider="exa_ai", - api_key="synthetic-exa-key", - api_base=f"{wire.url}/exa", - max_results=1, - contents={"highlights": {"numSentences": 2}}, - ) - assert len(wire.drain()) == 1 - - assert isinstance(response, SearchResponse), type(response) - assert response.model_dump(mode="json") == _EXPECTED +PARAMS: Final = { + "contents": {"highlights": {"dynamic": True, "verbosity": "low"}}, + "outputSchema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "systemPrompt": "Use evidence", + "type": "auto", +} @pytest.mark.asyncio -async def test_async_exa_search_returns_highlights_scores_and_the_response_envelope() -> None: - with wire_server(_respond) as wire: - response: Final = await litellm.asearch( - query=f"sdk async {uuid.uuid4().hex}", - search_provider="exa_ai", - api_key="synthetic-exa-key", - api_base=f"{wire.url}/exa", - max_results=1, - contents={"highlights": {"numSentences": 2}}, - ) - assert len(wire.drain()) == 1 +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("schema", [False, True]) +@pytest.mark.parametrize("price_override", [None, 0.0, 0.123]) +@pytest.mark.parametrize("global_callback", [False, True]) +async def test_exa_search_request_response_and_logged_cost_cross_real_http( + monkeypatch: pytest.MonkeyPatch, + asynchronous: bool, + schema: bool, + price_override: float | None, + global_callback: bool, +) -> None: + class Recorder(CustomLogger): + def __init__(self) -> None: + self.finished: Final = threading.Event() + self.records: tuple[tuple[Mapping[str, object], object], ...] = () - assert isinstance(response, SearchResponse), type(response) - assert response.model_dump(mode="json") == _EXPECTED + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.records = (*self.records, (kwargs, response_obj)) + self.finished.set() + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.log_success_event(kwargs, response_obj, start_time, end_time) + + recorder: Final = Recorder() + if global_callback: + monkeypatch.setattr(litellm, "callbacks", [recorder]) + provider_params: Final = { + **{key: value for key, value in PARAMS.items() if schema or key != "outputSchema"}, + "contents": PARAMS["contents"] if schema else {"highlights": {"numSentences": 2}}, + } + + def respond(request: Request) -> Reply: + assert request.headers["x-api-key"] == "test-key" + assert request.headers["exa-beta"] == "dynamic-highlights-2026-08-28" + assert json.loads(request.body) == {"query": "question", "numResults": 1, **provider_params} + return _respond(request) + + with wire_server(respond) as server: + params: Final = { + "query": "question", + "search_provider": "exa_ai", + "api_base": f"{server.url}/exa", + "max_results": 1, + "api_key": "test-key", + "extra_headers": {"Exa-Beta": "dynamic-highlights-2026-08-28"}, + **({"input_cost_per_query": price_override} if price_override is not None else {}), + **( + {} + if global_callback + else {"success_callback": [recorder.async_log_success_event if asynchronous else recorder]} + ), + **provider_params, + } + response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) + assert len(server.drain()) == 1 + assert isinstance(response, SearchResponse) + assert response.model_dump(mode="json") == _EXPECTED + assert await asyncio.to_thread(recorder.finished.wait, 5) + assert len(recorder.records) == 1 + details, logged = recorder.records[0] + assert isinstance(logged, SearchResponse) + assert logged.model_dump(mode="json") == _EXPECTED + assert isinstance(details["standard_logging_object"], Mapping) + expected_cost: Final = _REPORTED_COST if price_override is None else price_override + assert response.hidden_params["response_cost"] == expected_cost + assert details["response_cost"] == expected_cost + assert details["standard_logging_object"]["response_cost"] == expected_cost + assert details["stream"] is False diff --git a/tests/integration/sdk/test_exa_search_wire.py b/tests/integration/sdk/test_exa_search_wire.py deleted file mode 100644 index c27f0cfc548..00000000000 --- a/tests/integration/sdk/test_exa_search_wire.py +++ /dev/null @@ -1,105 +0,0 @@ -import json -import threading -from typing import Final - -import pytest - -import litellm -from tests.integration._support.wire import Reply, Request, wire_server - -PAYLOAD: Final = { - "results": [{"title": "Source", "url": "https://example.com", "highlights": ["evidence"], "author": None}], - "output": {"content": {"answer": "verified"}, "grounding": []}, - "requestId": "wire-request", - "searchTime": 14, - "costDollars": {"total": 0.01}, -} -PARAMS: Final = { - "contents": {"highlights": {"dynamic": True, "verbosity": "low"}}, - "outputSchema": {"type": "object", "properties": {"answer": {"type": "string"}}}, - "systemPrompt": "Use evidence", - "type": "auto", -} - - -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True]) -async def test_exa_search_request_and_full_response_cross_real_http(asynchronous: bool) -> None: - def respond(request: Request) -> Reply: - assert request.target == "/search" - assert request.headers["x-api-key"] == "test-key" - assert request.headers["exa-beta"] == "dynamic-highlights-2026-08-28" - assert json.loads(request.body) == {"query": "question", **PARAMS} - return Reply(body=json.dumps(PAYLOAD).encode()) - - with wire_server(respond) as server: - params: Final = { - "query": "question", - "search_provider": "exa_ai", - "api_base": server.url, - "api_key": "test-key", - "extra_headers": {"Exa-Beta": "dynamic-highlights-2026-08-28"}, - **PARAMS, - } - response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) - result: Final = json.loads(response.model_dump_json()) - assert result["provider_metadata"]["output"] == PAYLOAD["output"] - assert result["provider_metadata"]["requestId"] == PAYLOAD["requestId"] - assert result["provider_metadata"]["costDollars"] == PAYLOAD["costDollars"] - assert result["results"][0]["highlights"] == ["evidence"] - assert result["results"][0]["author"] is None - assert result["results"][0]["snippet"] == "evidence" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("asynchronous", [False, True]) -@pytest.mark.parametrize("schema", [False, True]) -@pytest.mark.parametrize("price_override", [None, 0.0, 0.123]) -async def test_search_logs_one_completed_response_and_cost( - asynchronous: bool, schema: bool, price_override: float | None -) -> None: - import asyncio - from collections.abc import Mapping - - from litellm.integrations.custom_logger import CustomLogger - - class Recorder(CustomLogger): - def __init__(self) -> None: - self.finished = threading.Event() - self.records = () - - def log_success_event(self, kwargs, response_obj, start_time, end_time): - self.records = (*self.records, (kwargs, response_obj)) - self.finished.set() - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - self.log_success_event(kwargs, response_obj, start_time, end_time) - - recorder: Final = Recorder() - - def respond(request: Request) -> Reply: - assert "input_cost_per_query" not in json.loads(request.body) - return Reply(body=json.dumps(PAYLOAD).encode()) - - with wire_server(respond) as server: - params: Final = { - "query": "question", - "search_provider": "exa_ai", - "api_base": server.url, - "api_key": "test-key", - **({"input_cost_per_query": price_override} if price_override is not None else {}), - "success_callback": [recorder.async_log_success_event if asynchronous else recorder], - **{key: value for key, value in PARAMS.items() if schema or key != "outputSchema"}, - } - response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) - assert await asyncio.to_thread(recorder.finished.wait, 5) - assert len(recorder.records) == 1 - details, logged = recorder.records[0] - assert ( - logged.model_dump()["provider_metadata"]["output"] == response.model_dump()["provider_metadata"]["output"] - ) - assert isinstance(details["standard_logging_object"], Mapping) - expected_cost: Final = PAYLOAD["costDollars"]["total"] if price_override is None else price_override - assert details["response_cost"] == expected_cost - assert details["standard_logging_object"]["response_cost"] == expected_cost - assert details["stream"] is False diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 389941dba4d..f5f88a6ee69 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -1,18 +1,13 @@ -import asyncio import json -import uuid -from collections.abc import Mapping from typing import Final from unittest.mock import Mock import httpx import pytest import respx -from pydantic import TypeAdapter, ValidationError +from pydantic import ValidationError import litellm -from litellm.integrations.custom_logger import CustomLogger -from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_COST_DOLLARS_PARAM from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig @@ -20,7 +15,6 @@ from litellm.search.cost_calculator import search_provider_cost_per_query EXA_SEARCH_URL: Final = "https://api.exa.ai/search" EXA_REPORTED_COST: Final = 0.123456 -_LOGGED: Final = TypeAdapter(Mapping[str, object]) @pytest.fixture @@ -385,53 +379,21 @@ def test_a_caller_cannot_set_the_cost_exa_reports(respx_mock: respx.MockRouter) @pytest.mark.usefixtures("exa_api_key") -def test_costing_an_exa_reply_directly_returns_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: +@pytest.mark.parametrize("serialized", (False, True)) +def test_costing_an_exa_reply_directly_returns_the_cost_exa_reports( + respx_mock: respx.MockRouter, serialized: bool +) -> None: respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}) response: Final = litellm.search(query="asyncio TaskGroup", search_provider="exa_ai", max_results=25) assert _catalog_price(25) != EXA_REPORTED_COST + costed_response: Final = response.model_dump() if serialized else response completion_cost: Final = litellm.completion_cost( - completion_response=response, model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search" + completion_response=costed_response, model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search" ) per_token: Final = litellm.cost_per_token( - model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search", response=response + model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search", response=costed_response ) assert completion_cost == pytest.approx(EXA_REPORTED_COST) assert per_token == pytest.approx((EXA_REPORTED_COST, 0.0)) - - -class _SpendCapture(CustomLogger): - def __init__(self, call_id: str) -> None: - super().__init__() - self.call_id: Final = call_id - self.costs: Final[list[object]] = [] # mutable-ok: the logging worker appends one entry per logged call - - async def async_log_success_event( - self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object - ) -> None: - if kwargs.get("litellm_call_id") == self.call_id: - self.costs.append(_LOGGED.validate_python(kwargs.get("standard_logging_object") or {}).get("response_cost")) - - -@pytest.mark.asyncio -@pytest.mark.usefixtures("exa_api_key") -async def test_async_search_logs_the_cost_exa_reports_as_spend( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - litellm.in_memory_llm_clients_cache.flush_cache() - GLOBAL_LOGGING_WORKER.start() - capture: Final = _SpendCapture(call_id=f"exa-{uuid.uuid4()}") - monkeypatch.setattr(litellm, "callbacks", [capture]) - respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [], "costDollars": {"total": EXA_REPORTED_COST}}) - assert _catalog_price(25) != EXA_REPORTED_COST - - response: Final = await litellm.asearch( - query="asyncio TaskGroup", search_provider="exa_ai", max_results=25, litellm_call_id=capture.call_id - ) - await asyncio.sleep(0) - await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) - - assert response.hidden_params["response_cost"] == EXA_REPORTED_COST - assert capture.costs == [EXA_REPORTED_COST] diff --git a/tests/unit/search/test_main.py b/tests/unit/search/test_main.py index 5ae3b25fcf8..e1f1330fa7e 100644 --- a/tests/unit/search/test_main.py +++ b/tests/unit/search/test_main.py @@ -1,7 +1,6 @@ import datetime import json -from collections.abc import Callable, Iterator, Mapping -from dataclasses import dataclass +from collections.abc import Iterator, Mapping from typing import Final import httpx @@ -64,8 +63,13 @@ def _proxy_style_logging_obj() -> Logging: @pytest.mark.usefixtures("_local_model_cost_map") -def test_search_bills_search_tool_input_cost_per_query(respx_mock: respx.MockRouter) -> None: - respx_mock.post(EXA_SEARCH_URL).respond(json={"results": [{"title": "t", "url": "https://example.com"}]}) +@pytest.mark.parametrize("reported_cost", (None, EXA_REPORTED_COST)) +def test_search_bills_search_tool_input_cost_per_query( + respx_mock: respx.MockRouter, reported_cost: float | None +) -> None: + respx_mock.post(EXA_SEARCH_URL).respond( + json={"results": [], **({"costDollars": {"total": reported_cost}} if reported_cost is not None else {})} + ) map_price, _ = search_provider_cost_per_query(model="exa_ai/search", custom_llm_provider="exa_ai") configured_price: Final = 0.05 assert map_price not in (0.0, configured_price) @@ -84,19 +88,29 @@ def test_search_bills_search_tool_input_cost_per_query(respx_mock: respx.MockRou litellm_logging_obj=_proxy_style_logging_obj(), ) - assert map_priced._hidden_params["response_cost"] == pytest.approx(map_price) + assert map_priced._hidden_params["response_cost"] == pytest.approx( + map_price if reported_cost is None else reported_cost + ) assert tool_priced._hidden_params["response_cost"] == pytest.approx(configured_price) assert respx_mock.calls.last.request.content.find(b"input_cost_per_query") == -1 @pytest.mark.usefixtures("_local_model_cost_map") +@pytest.mark.parametrize("reported_cost", (None, EXA_REPORTED_COST)) def test_unpriced_fallback_search_tool_is_billed_at_its_own_price_not_the_failed_primarys( respx_mock: respx.MockRouter, + reported_cost: float | None, ) -> None: respx_mock.post(EXA_SEARCH_URL).mock( side_effect=( httpx.Response(500, json={"error": "primary down"}), - httpx.Response(200, json={"results": [{"title": "t", "url": "https://example.com"}]}), + httpx.Response( + 200, + json={ + "results": [], + **({"costDollars": {"total": reported_cost}} if reported_cost is not None else {}), + }, + ), ) ) map_price, _ = search_provider_cost_per_query(model="exa_ai/search", custom_llm_provider="exa_ai") @@ -119,103 +133,22 @@ def test_unpriced_fallback_search_tool_is_billed_at_its_own_price_not_the_failed litellm_logging_obj=shared_by_every_fallback_attempt, ) - assert fallback._hidden_params["response_cost"] == pytest.approx(map_price) - - -@pytest.mark.usefixtures("_local_model_cost_map") -def test_a_search_tool_price_overrides_the_cost_exa_reports(respx_mock: respx.MockRouter) -> None: - respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) - configured_price: Final = 0.05 - assert configured_price != EXA_REPORTED_COST - - reported: Final = litellm.search( - query="litellm release notes", - search_provider="exa_ai", - api_key="test-exa-key", - litellm_logging_obj=_proxy_style_logging_obj(), - ) - tool_priced: Final = litellm.search( - query="litellm release notes", - search_provider="exa_ai", - api_key="test-exa-key", - input_cost_per_query=configured_price, - litellm_logging_obj=_proxy_style_logging_obj(), - ) - - assert reported.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) - assert tool_priced.hidden_params["response_cost"] == pytest.approx(configured_price) - - -@pytest.mark.usefixtures("_local_model_cost_map") -def test_unpriced_fallback_search_tool_bills_the_cost_exa_reports_not_the_failed_primarys_price( - respx_mock: respx.MockRouter, -) -> None: - respx_mock.post(EXA_SEARCH_URL).mock( - side_effect=(httpx.Response(500, json={"error": "primary down"}), httpx.Response(200, json=EXA_REPLY_WITH_COST)) - ) - primary_price: Final = 0.05 - shared_by_every_fallback_attempt: Final = _proxy_style_logging_obj() - - with pytest.raises(litellm.InternalServerError): - litellm.search( - query="litellm release notes", - search_provider="exa_ai", - api_key="test-exa-key", - input_cost_per_query=primary_price, - litellm_logging_obj=shared_by_every_fallback_attempt, - ) - fallback: Final = litellm.search( - query="litellm release notes", - search_provider="exa_ai", - api_key="test-exa-key", - litellm_logging_obj=shared_by_every_fallback_attempt, - ) - - assert fallback.hidden_params["response_cost"] == pytest.approx(EXA_REPORTED_COST) - - -def _single_tier(price: float) -> list[dict[str, object]]: - return [{"input_cost_per_query": price, "max_results_range": [0, 100]}] - - -def _register_exa_price(monkeypatch: pytest.MonkeyPatch, price: float) -> None: - litellm.register_model( - {"exa_ai/search": {"input_cost_per_query": price, "litellm_provider": "exa_ai", "mode": "search"}} - ) - - -def _register_exa_tiers(monkeypatch: pytest.MonkeyPatch, price: float) -> None: - litellm.register_model( - {"exa_ai/search": {"tiered_pricing": _single_tier(price), "litellm_provider": "exa_ai", "mode": "search"}} - ) - - -def _serve_exa_price_in_cost_map(monkeypatch: pytest.MonkeyPatch, price: float) -> None: - monkeypatch.setitem( - litellm.model_cost, "exa_ai/search", {**litellm.model_cost["exa_ai/search"], "input_cost_per_query": price} - ) - - -def _serve_exa_tiers_in_cost_map(monkeypatch: pytest.MonkeyPatch, price: float) -> None: - monkeypatch.setitem( - litellm.model_cost, - "exa_ai/search", - {**litellm.model_cost["exa_ai/search"], "tiered_pricing": _single_tier(price)}, + assert fallback._hidden_params["response_cost"] == pytest.approx( + map_price if reported_cost is None else reported_cost ) @pytest.mark.usefixtures("_local_model_cost_map") -@pytest.mark.parametrize( - "set_exa_price", - [_register_exa_price, _register_exa_tiers, _serve_exa_price_in_cost_map, _serve_exa_tiers_in_cost_map], -) +@pytest.mark.parametrize("registered", (False, True)) +@pytest.mark.parametrize("price_field", ("input_cost_per_query", "tiered_pricing")) @pytest.mark.parametrize( "exa_reply", [EXA_REPLY_WITH_COST, {"results": []}], ids=["exa-reports-a-cost", "exa-reports-no-cost"] ) def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_litellms_tiers( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, - set_exa_price: Callable[[pytest.MonkeyPatch, float], None], + registered: bool, + price_field: str, exa_reply: Mapping[str, object], ) -> None: respx_mock.post(EXA_SEARCH_URL).respond(json=exa_reply) @@ -224,7 +157,17 @@ def test_an_exa_price_someone_set_overrides_the_cost_exa_reports_and_litellms_ti ) set_price: Final = 0.003 assert set_price not in (tier_price, EXA_REPORTED_COST) - set_exa_price(monkeypatch, set_price) + price: Final = ( + set_price + if price_field == "input_cost_per_query" + else [{"input_cost_per_query": set_price, "max_results_range": [0, 100]}] + ) + if registered: + litellm.register_model({"exa_ai/search": {price_field: price, "litellm_provider": "exa_ai", "mode": "search"}}) + else: + monkeypatch.setitem( + litellm.model_cost, "exa_ai/search", {**litellm.model_cost["exa_ai/search"], price_field: price} + ) response: Final = litellm.search( query="litellm release notes", search_provider="exa_ai", api_key="test-exa-key", max_results=25 @@ -456,19 +399,6 @@ def test_a_runtime_price_on_the_mode_entry_wins_over_a_runtime_price_on_the_gene assert cost == pytest.approx((0.011, 0.0)) -@pytest.mark.usefixtures("_local_model_cost_map") -def test_exa_receipt_survives_public_response_serialization(respx_mock: respx.MockRouter) -> None: - respx_mock.post(EXA_SEARCH_URL).respond(json=EXA_REPLY_WITH_COST) - response: Final = litellm.search(query="receipt", search_provider="exa_ai", api_key="test-key") - - assert litellm.completion_cost( - completion_response=response.model_dump(), - model="exa_ai/search", - custom_llm_provider="exa_ai", - call_type="search", - ) == pytest.approx(EXA_REPORTED_COST) - - @pytest.mark.usefixtures("_local_model_cost_map") @pytest.mark.parametrize("params", ({"type": "deep"}, {}), ids=("explicit-mode", "default-mode")) def test_exa_pricing_apis_keep_the_requested_mode(respx_mock: respx.MockRouter, params: Mapping[str, object]) -> None: @@ -513,14 +443,6 @@ def test_exa_native_result_count_bills_the_same_operator_tier(respx_mock: respx. assert response.hidden_params["response_cost"] == pytest.approx(price) -@dataclass(frozen=True, slots=True) -class _CostMapServer: - body: bytes - - def get(self, url: str, *, timeout: float | None = None) -> httpx.Response: - return httpx.Response(200, content=self.body, request=httpx.Request("GET", url)) - - @pytest.fixture def _restore_catalog_source(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: yield @@ -556,11 +478,12 @@ def test_exa_catalog_source_preserves_receipts_and_generic_operator_prices( **({"input_cost_per_query": price} if price is not None else {}), }, } + respx_mock.get(source).respond(json=served) monkeypatch.delenv("LITELLM_LOCAL_MODEL_COST_MAP", raising=False) monkeypatch.setattr( litellm, "model_cost", - litellm.get_model_cost_map(url=source, max_attempts=1, client=_CostMapServer(json.dumps(served).encode())), + litellm.get_model_cost_map(url=source, max_attempts=1), ) response: Final = litellm.search(query="catalog", search_provider="exa_ai", api_key="test-key", type=mode) assert response.hidden_params["response_cost"] == pytest.approx(expected_cost)