diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 8331c68fe24..0f82b137e76 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -709,6 +709,7 @@ def cost_per_token( custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries or 1, optional_params=(getattr(response, "_hidden_params", None) if response else None), + response=response, ) elif ( per_second_cost := _per_second_pricing_cost( @@ -1914,6 +1915,7 @@ def completion_cost( custom_llm_provider=custom_llm_provider, number_of_queries=number_of_queries, optional_params=optional_params, + response=cast(object, completion_response), # cast-ok: callee validates legacy dict fields ) ) diff --git a/litellm/llms/exa_ai/search/cost_calculator.py b/litellm/llms/exa_ai/search/cost_calculator.py new file mode 100644 index 00000000000..4b1d561cf47 --- /dev/null +++ b/litellm/llms/exa_ai/search/cost_calculator.py @@ -0,0 +1,53 @@ +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +from litellm.utils import get_model_info + +EXA_AI_INCLUDED_RESULTS: Final = 10 +EXA_AI_ADDITIONAL_RESULT_COST: Final = 0.001 +EXA_AI_STANDARD_SEARCH_MODEL: Final = "exa_ai/search" +EXA_AI_PRICING_MODEL_BY_TYPE: Final[Mapping[str, str]] = MappingProxyType( + { + "lite": "exa_ai/search-lite", + "instant": "exa_ai/search-instant", + "deep-lite": "exa_ai/search-deep", + "deep": "exa_ai/search-deep", + "deep-reasoning": "exa_ai/search-deep-reasoning", + } +) + + +def _non_negative_int(value: object) -> int | None: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + return None + return value + + +def _requested_results(optional_params: Mapping[str, object]) -> int: + for key in ("max_results", "numResults"): + requested = _non_negative_int(optional_params.get(key)) + if requested is not None: + return requested + return EXA_AI_INCLUDED_RESULTS + + +def _base_request_cost(search_type: object) -> float: + pricing_model: Final = ( + EXA_AI_PRICING_MODEL_BY_TYPE.get(search_type, EXA_AI_STANDARD_SEARCH_MODEL) + if isinstance(search_type, str) + else EXA_AI_STANDARD_SEARCH_MODEL + ) + model_info: Final = get_model_info(model=pricing_model, custom_llm_provider="exa_ai") + return float(model_info.get("input_cost_per_query") or 0.0) + + +def exa_ai_list_price(optional_params: Mapping[str, object]) -> float: + """ + List price of one Exa /search request, for responses that carry no `costDollars.total`. + + Multi-query searches are joined into a single request, so this prices one request: + the per-type base price plus a per-result charge above the results that price includes. + """ + additional_results: Final = max(_requested_results(optional_params) - EXA_AI_INCLUDED_RESULTS, 0) + return _base_request_cost(optional_params.get("type")) + additional_results * EXA_AI_ADDITIONAL_RESULT_COST diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 73e341d0033..970ef2a03a9 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -1,11 +1,11 @@ """ Calls Exa AI's /search endpoint to search the web. -Exa AI API Reference: https://docs.exa.ai/reference/search +Exa AI API Reference: https://exa.ai/docs/reference/search """ -from collections.abc import Iterable, Mapping -from typing import Final, TypedDict +from collections.abc import Iterable, Mapping, Sequence +from typing import Final import httpx from pydantic import ConfigDict, TypeAdapter @@ -17,43 +17,17 @@ from litellm.llms.base_llm.search.transformation import ( SearchResponse, SearchResult, ) +from litellm.llms.exa_ai.search.types import ExaAISearchRequest from litellm.secret_managers.main import get_secret_str +__all__ = ["ExaAISearchConfig", "ExaAISearchRequest"] + _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True)) _STRS: Final = TypeAdapter(Iterable[str], config=ConfigDict(hide_input_in_errors=True)) _RESULT_EXTRAS: Final = (("highlights", "highlights"), ("highlightScores", "highlight_scores"), ("score", "score")) -class _ExaAISearchRequestRequired(TypedDict): - """Required fields for Exa AI Search API request.""" - - query: str # Required - search query - - -class ExaAISearchRequest(_ExaAISearchRequestRequired, total=False): - """ - Exa AI Search API request format. - Based on: https://docs.exa.ai/reference/search - """ - - type: str # Optional - search type ('keyword', 'neural', 'fast', 'auto'), default 'auto' - category: str # Optional - data category ('company', 'research paper', 'news', 'pdf', 'github', 'tweet', 'personal site', 'linkedin profile', 'financial report') - userLocation: str # Optional - two-letter ISO country code - numResults: int # Optional - number of results (max 100), default 10 - includeDomains: list[str] # Optional - list of domains to include - excludeDomains: list[str] # Optional - list of domains to exclude - startCrawlDate: str # Optional - crawl date filter (ISO 8601 format) - endCrawlDate: str # Optional - crawl date filter (ISO 8601 format) - startPublishedDate: str # Optional - published date filter (ISO 8601 format) - endPublishedDate: str # Optional - published date filter (ISO 8601 format) - includeText: list[str] # Optional - strings that must be present in webpage text - excludeText: list[str] # Optional - strings that must not be present in webpage text - context: bool | dict # Optional - format results for LLMs - moderation: bool # Optional - enable content moderation, default false - contents: dict # Optional - content retrieval options - - class ExaAISearchConfig(BaseSearchConfig): EXA_AI_API_BASE = "https://api.exa.ai" @@ -63,33 +37,31 @@ class ExaAISearchConfig(BaseSearchConfig): def validate_environment( self, - headers: dict, + headers: Mapping[str, object], api_key: str | None = None, api_base: str | None = None, - **kwargs, - ) -> dict: + **kwargs: object, + ) -> dict[str, object]: """ Validate environment and return headers. """ - api_key = self.resolve_server_api_key( + resolved_key: Final = self.resolve_server_api_key( caller_api_key=api_key, caller_api_base=api_base, key_env_vars=("EXA_API_KEY",), base_env_var="EXA_API_BASE", default_api_base=self.EXA_AI_API_BASE, ) - if not api_key: + if not resolved_key: raise ValueError("EXA_API_KEY is not set. Set `EXA_API_KEY` environment variable.") - headers["x-api-key"] = api_key - headers["Content-Type"] = "application/json" - return headers + return {**headers, "x-api-key": resolved_key, "Content-Type": "application/json"} def get_complete_url( self, api_base: str | None, - optional_params: dict, - data: dict | list[dict] | None = None, - **kwargs, + optional_params: Mapping[str, object], + data: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None, + **kwargs: object, ) -> str: """ Get complete URL for Search endpoint. @@ -99,9 +71,9 @@ class ExaAISearchConfig(BaseSearchConfig): def transform_search_request( self, query: str | list[str], - optional_params: dict, - **kwargs, - ) -> dict: + optional_params: Mapping[str, object], + **kwargs: object, + ) -> dict[str, object]: """ Transform Search request to Exa AI API format. @@ -121,44 +93,32 @@ class ExaAISearchConfig(BaseSearchConfig): Returns: Dict with typed request data following ExaAISearchRequest spec """ - if isinstance(query, list): - # Exa AI only supports single string queries, join with spaces - query = " ".join(query) - - request_data: Final[ExaAISearchRequest] = { - "query": query, + mapped: Final = { + target: optional_params[source] + for source, target in ( + ("max_results", "numResults"), + ("search_domain_filter", "includeDomains"), + ("country", "userLocation"), + ) + if source in optional_params + } + provider_params: Final = { + key: value + for key, value in optional_params.items() + if key not in self.get_supported_perplexity_optional_params() and key not in mapped + } + return { + "query": " ".join(query) if isinstance(query, list) else query, + **mapped, + **provider_params, + **({"contents": {"text": True}} if "contents" not in provider_params else {}), } - - # Transform Perplexity unified spec parameters to Exa format - if "max_results" in optional_params: - request_data["numResults"] = optional_params["max_results"] - - if "search_domain_filter" in optional_params: - request_data["includeDomains"] = optional_params["search_domain_filter"] - - if "country" in optional_params: - request_data["userLocation"] = optional_params["country"] - - # Convert to dict before dynamic key assignments - result_data: Final = dict(request_data) - - # pass through all other parameters as-is - for param, value in optional_params.items(): - if param not in self.get_supported_perplexity_optional_params() and param not in result_data: - result_data[param] = value - - # By default, request text content if not explicitly specified - # Exa AI doesn't return content/text unless explicitly requested - if "contents" not in result_data: - result_data["contents"] = {"text": True} - - return result_data def transform_search_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, - **kwargs, + **kwargs: object, ) -> SearchResponse: """ Transform Exa AI API response to LiteLLM unified SearchResponse format. @@ -184,29 +144,32 @@ class ExaAISearchConfig(BaseSearchConfig): """ response_json: Final = _JSON_OBJECT.validate_python(raw_response.json()) - # Transform results to SearchResult objects - results: Final = [] - for raw_result in _JSON_VALUES.validate_python(response_json.get("results", [])): - result = _JSON_OBJECT.validate_python(raw_result) - search_result = SearchResult.model_validate( - { - "title": result.get("title", ""), - "url": result.get("url", ""), - "snippet": result.get("text") - or "\n\n".join(_STRS.validate_python(result.get("highlights") or [])) - or result.get("summary") - or "", - "date": result.get("publishedDate"), # ISO 8601 datetime string - "last_updated": None, # Exa AI doesn't provide last_updated in response - **{field: result[key] for key, field in _RESULT_EXTRAS if key in result}, - } - ) - results.append(search_result) - return SearchResponse.model_validate( { - "results": results, + **response_json, + "results": [ + self._transform_result(value) + for value in _JSON_VALUES.validate_python(response_json.get("results") or []) + ], "object": "search", "provider_metadata": {key: value for key, value in response_json.items() if key != "results"}, } ) + + @staticmethod + def _transform_result(value: object) -> SearchResult: + result: Final = _JSON_OBJECT.validate_python(value) + return SearchResult.model_validate( + { + **result, + "title": result.get("title", ""), + "url": result.get("url", ""), + "snippet": result.get("text") + or "\n\n".join(_STRS.validate_python(result.get("highlights") or [])) + or result.get("summary") + or "", + "date": result.get("publishedDate"), + "last_updated": None, + **{field: result[key] for key, field in _RESULT_EXTRAS if key in result}, + } + ) diff --git a/litellm/llms/exa_ai/search/types.py b/litellm/llms/exa_ai/search/types.py new file mode 100644 index 00000000000..3fd1379701f --- /dev/null +++ b/litellm/llms/exa_ai/search/types.py @@ -0,0 +1,99 @@ +"""Exa /search request contract, https://exa.ai/docs/reference/search.""" + +from collections.abc import Mapping, Sequence +from typing import Literal, TypeAlias + +from typing_extensions import ReadOnly, Required, TypedDict + +ExaSearchMode: TypeAlias = Literal["lite", "instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"] +ExaPageSection: TypeAlias = Literal["header", "navigation", "banner", "body", "sidebar", "footer", "metadata"] + + +class ExaTextOptions(TypedDict, total=False): + maxCharacters: ReadOnly[int | None] + includeHtmlTags: ReadOnly[bool | None] + verbosity: ReadOnly[Literal["compact", "standard", "full"] | None] + includeSections: ReadOnly[Sequence[ExaPageSection] | None] + excludeSections: ReadOnly[Sequence[ExaPageSection] | None] + + +class ExaHighlightsOptions(TypedDict, total=False): + """numSentences and highlightsPerUrl are deprecated compatibility fields.""" + + query: ReadOnly[str | None] + verbosity: ReadOnly[Literal["low", "medium", "high"] | None] + dynamic: ReadOnly[bool | None] + maxCharacters: ReadOnly[int | None] + numSentences: ReadOnly[int | None] + highlightsPerUrl: ReadOnly[int | None] + + +class ExaSummaryOptions(TypedDict, total=False): + query: ReadOnly[str | None] + schema: ReadOnly[Mapping[str, object] | None] + + +class ExaExtrasOptions(TypedDict, total=False): + links: ReadOnly[int | None] + imageLinks: ReadOnly[int | None] + richImageLinks: ReadOnly[int | None] + richLinks: ReadOnly[int | None] + codeBlocks: ReadOnly[int | None] + + +class ExaContextOptions(TypedDict, total=False): + maxCharacters: ReadOnly[int] + + +class ExaContentsOptions(TypedDict, total=False): + """context, livecrawl, and crawledBeforeDate are deprecated compatibility fields.""" + + text: ReadOnly[bool | ExaTextOptions | None] + highlights: ReadOnly[bool | ExaHighlightsOptions | None] + summary: ReadOnly[ExaSummaryOptions | None] + extras: ReadOnly[ExaExtrasOptions | None] + livecrawlTimeout: ReadOnly[int | None] + maxAgeHours: ReadOnly[int | None] + snapshotAsOf: ReadOnly[str | None] + subpages: ReadOnly[int | None] + subpageTarget: ReadOnly[str | Sequence[str] | None] + context: ReadOnly[bool | ExaContextOptions | None] + livecrawl: ReadOnly[Literal["never", "always", "fallback", "preferred"] | None] + crawledBeforeDate: ReadOnly[str | None] + + +class ExaCountryLocation(TypedDict): + country: ReadOnly[str] + + +class ExaCoordinateLocation(TypedDict, total=False): + country: ReadOnly[str] + latitude: Required[ReadOnly[float]] + longitude: Required[ReadOnly[float]] + + +class ExaAISearchRequest(TypedDict, total=False): + """Crawl-date filters, includeText, excludeText, and context are deprecated.""" + + query: Required[ReadOnly[str]] + type: ReadOnly[ExaSearchMode | None] + category: ReadOnly[str | None] + additionalQueries: ReadOnly[Sequence[str] | None] + userLocation: ReadOnly[str | ExaCountryLocation | ExaCoordinateLocation | None] + numResults: ReadOnly[int | None] + includeDomains: ReadOnly[Sequence[str] | None] + excludeDomains: ReadOnly[Sequence[str] | None] + startPublishedDate: ReadOnly[str | None] + endPublishedDate: ReadOnly[str | None] + moderation: ReadOnly[bool | None] + contents: ReadOnly[ExaContentsOptions | None] + outputSchema: ReadOnly[Mapping[str, object] | None] + objective: ReadOnly[str | None] + systemPrompt: ReadOnly[str | None] + userIdentifier: ReadOnly[str | None] + compliance: ReadOnly[Literal["hipaa"] | None] + startCrawlDate: ReadOnly[str | None] + endCrawlDate: ReadOnly[str | None] + includeText: ReadOnly[Sequence[str] | None] + excludeText: ReadOnly[Sequence[str] | None] + context: ReadOnly[bool | ExaContextOptions | None] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9a7d88b8eda..d5f59e79018 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24461,24 +24461,44 @@ "output_vector_size": 2560 }, "exa_ai/search": { + "input_cost_per_query": 0.007, "litellm_provider": "exa_ai", - "mode": "search", - "tiered_pricing": [ - { - "input_cost_per_query": 0.005, - "max_results_range": [ - 0, - 25 - ] - }, - { - "input_cost_per_query": 0.025, - "max_results_range": [ - 26, - 100 - ] - } - ] + "metadata": { + "notes": "Base price per /search request, covering up to 10 results. Each result above 10 adds $0.001. Exa's reported costDollars.total takes precedence when present. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-lite": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-22)" + }, + "mode": "search" + }, + "exa_ai/search-instant": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type instant, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep or deep-lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep-reasoning": { + "input_cost_per_query": 0.015, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-reasoning, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" }, "firecrawl/search": { "litellm_provider": "firecrawl", diff --git a/litellm/search/cost_calculator.py b/litellm/search/cost_calculator.py index 15755128136..746f470a8ba 100644 --- a/litellm/search/cost_calculator.py +++ b/litellm/search/cost_calculator.py @@ -2,6 +2,7 @@ Cost calculation for search providers. """ +import math from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -10,11 +11,13 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.utils import get_model_info PROVIDER_USAGE_ADAPTER: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter( tuple[Mapping[str, object], ...] ) +PROVIDER_COST_ADAPTER: Final = TypeAdapter(Mapping[str, object]) EMPTY_OPTIONAL_PARAMS: Final[Mapping[str, object]] = MappingProxyType({}) SEARCH_MODE_PARAM_BY_PROVIDER: Final[Mapping[str, str]] = MappingProxyType( {"tavily": "search_depth", "linkup": "depth"} @@ -64,11 +67,15 @@ def search_provider_cost_per_query( custom_llm_provider: str | None = None, number_of_queries: int = 1, optional_params: Mapping[str, object] | None = None, + response: object = None, ) -> tuple[float, float]: """ Calculate cost for search-only providers. Returns (input_cost, output_cost) where input_cost = queries * cost_per_query + Uses Exa's reported cost when available, otherwise its list price; other providers use tiered pricing based on max_results. + + Supports tiered pricing based on max_results parameter, and per-mode price entries selected by the provider's mode parameter (Tavily `search_depth`, Linkup `depth`). @@ -77,10 +84,26 @@ def search_provider_cost_per_query( custom_llm_provider: Provider name (e.g., "exa_ai", "tavily") number_of_queries: Number of search queries performed (default: 1) optional_params: Optional parameters including max_results for tiered pricing + response: Provider response with reported Exa costDollars.total, when available Returns: Tuple of (input_cost, output_cost) where output_cost is always 0.0 """ + if custom_llm_provider == "exa_ai": + payload: Final = response.model_extra if isinstance(response, SearchResponse) else response + reported: Final = ( + PROVIDER_COST_ADAPTER.validate_python(payload).get("costDollars") if isinstance(payload, Mapping) else None + ) + total: Final = ( + PROVIDER_COST_ADAPTER.validate_python(reported).get("total") if isinstance(reported, Mapping) else None + ) + if isinstance(total, (int, float)) and not isinstance(total, bool) and math.isfinite(total) and total >= 0: + return (float(total), 0.0) + + from litellm.llms.exa_ai.search.cost_calculator import exa_ai_list_price + + return (exa_ai_list_price(optional_params if optional_params is not None else EMPTY_OPTIONAL_PARAMS), 0.0) + if custom_llm_provider == "parallel_ai": from litellm.llms.parallel_ai.search.cost_calculator import ( PARALLEL_AI_USAGE_PARAM, @@ -98,7 +121,7 @@ def search_provider_cost_per_query( custom_llm_provider=custom_llm_provider, ) - # Check for tiered pricing (e.g., Exa AI based on max_results) + # Check for tiered pricing based on max_results tiered_pricing: Final = model_info.get("tiered_pricing") if tiered_pricing and isinstance(tiered_pricing, list): max_results: Final = (optional_params or {}).get("max_results", 10) # default 10 results diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9a7d88b8eda..d5f59e79018 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24461,24 +24461,44 @@ "output_vector_size": 2560 }, "exa_ai/search": { + "input_cost_per_query": 0.007, "litellm_provider": "exa_ai", - "mode": "search", - "tiered_pricing": [ - { - "input_cost_per_query": 0.005, - "max_results_range": [ - 0, - 25 - ] - }, - { - "input_cost_per_query": 0.025, - "max_results_range": [ - 26, - 100 - ] - } - ] + "metadata": { + "notes": "Base price per /search request, covering up to 10 results. Each result above 10 adds $0.001. Exa's reported costDollars.total takes precedence when present. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-lite": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-22)" + }, + "mode": "search" + }, + "exa_ai/search-instant": { + "input_cost_per_query": 0.004, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type instant, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep": { + "input_cost_per_query": 0.012, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep or deep-lite, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" + }, + "exa_ai/search-deep-reasoning": { + "input_cost_per_query": 0.015, + "litellm_provider": "exa_ai", + "metadata": { + "notes": "Base price per /search request with type deep-reasoning, covering up to 10 results. Source: https://exa.ai/pricing (2026-10-15)" + }, + "mode": "search" }, "firecrawl/search": { "litellm_provider": "firecrawl", diff --git a/tests/integration/sdk/test_exa_search_wire.py b/tests/integration/sdk/test_exa_search_wire.py new file mode 100644 index 00000000000..ef1cc703274 --- /dev/null +++ b/tests/integration/sdk/test_exa_search_wire.py @@ -0,0 +1,103 @@ +import json +import threading +from typing import Final + +import pytest + +import litellm +from tests.integration._support.wire import Reply, Request, wire_server + +PAYLOAD: Final = { + "results": [{"title": "Source", "url": "https://example.com", "highlights": ["evidence"], "author": None}], + "output": {"content": {"answer": "verified"}, "grounding": []}, + "requestId": "wire-request", + "searchTime": 14, + "costDollars": {"total": 0.01}, +} +PARAMS: Final = { + "contents": {"highlights": {"dynamic": True, "verbosity": "low"}}, + "outputSchema": {"type": "object", "properties": {"answer": {"type": "string"}}}, + "systemPrompt": "Use evidence", + "type": "auto", +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_exa_search_request_and_full_response_cross_real_http(asynchronous: bool) -> None: + def respond(request: Request) -> Reply: + assert request.target == "/search" + assert request.headers["x-api-key"] == "test-key" + assert request.headers["exa-beta"] == "dynamic-highlights-2026-08-28" + assert json.loads(request.body) == {"query": "question", **PARAMS} + return Reply(body=json.dumps(PAYLOAD).encode()) + + with wire_server(respond) as server: + params: Final = { + "query": "question", + "search_provider": "exa_ai", + "api_base": server.url, + "api_key": "test-key", + "extra_headers": {"Exa-Beta": "dynamic-highlights-2026-08-28"}, + **PARAMS, + } + response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) + result: Final = json.loads(response.model_dump_json()) + assert result["output"] == PAYLOAD["output"] + assert result["requestId"] == PAYLOAD["requestId"] + assert result["costDollars"] == PAYLOAD["costDollars"] + assert result["results"][0]["highlights"] == ["evidence"] + assert result["results"][0]["author"] is None + assert result["results"][0]["snippet"] == "evidence" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("schema", [False, True]) +@pytest.mark.parametrize("price_override", [None, 0.0, 0.123]) +async def test_search_logs_one_completed_response_and_cost( + asynchronous: bool, schema: bool, price_override: float | None +) -> None: + import asyncio + from collections.abc import Mapping + + from litellm.integrations.custom_logger import CustomLogger + + class Recorder(CustomLogger): + def __init__(self) -> None: + self.finished = threading.Event() + self.records = () + + def log_success_event(self, kwargs, response_obj, start_time, end_time): + self.records = (*self.records, (kwargs, response_obj)) + self.finished.set() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + self.log_success_event(kwargs, response_obj, start_time, end_time) + + recorder: Final = Recorder() + + def respond(request: Request) -> Reply: + assert "input_cost_per_query" not in json.loads(request.body) + return Reply(body=json.dumps(PAYLOAD).encode()) + + with wire_server(respond) as server: + params: Final = { + "query": "question", + "search_provider": "exa_ai", + "api_base": server.url, + "api_key": "test-key", + **({"input_cost_per_query": price_override} if price_override is not None else {}), + "success_callback": [recorder.async_log_success_event if asynchronous else recorder], + **{key: value for key, value in PARAMS.items() if schema or key != "outputSchema"}, + } + response: Final = await litellm.asearch(**params) if asynchronous else litellm.search(**params) + assert await asyncio.to_thread(recorder.finished.wait, 5) + assert len(recorder.records) == 1 + details, logged = recorder.records[0] + assert logged.model_dump()["output"] == response.model_dump()["output"] + assert isinstance(details["standard_logging_object"], Mapping) + expected_cost: Final = PAYLOAD["costDollars"]["total"] if price_override is None else price_override + assert details["response_cost"] == expected_cost + assert details["standard_logging_object"]["response_cost"] == expected_cost + assert details["stream"] is False diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index a5a5d5e22dc..0e65832d22f 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -122,35 +122,17 @@ def test_transform_search_response_maps_every_exa_result_in_provider_order(): ) assert response.object == "search" - assert response.results == [ - SearchResult.model_validate( - { - "title": "First", - "url": "https://example.com/first", - "snippet": "First text", - "date": "2024-01-05T00:00:00.000Z", - "highlights": ["ignored when text is present"], - "highlight_scores": [0.5], - "score": 0.42, - } - ), - SearchResult.model_validate( - { - "title": "Second", - "url": "https://example.com/second", - "snippet": "one\n\ntwo", - "highlights": ["one", "two"], - } + expected: Final = [ + SearchResult( + title="First", url="https://example.com/first", snippet="First text", date="2024-01-05T00:00:00.000Z" ), + SearchResult(title="Second", url="https://example.com/second", snippet="one\n\ntwo"), SearchResult(title="Third", url="https://example.com/third", snippet="Summary"), ] - assert response.model_extra == { - "provider_metadata": { - "requestId": "b5947044c4b78efa9552a7c89b306d95", - "resolvedSearchType": "neural", - "costDollars": {"total": 0.005}, - } - } + + assert [result.model_dump(include=set(SearchResult.model_fields)) for result in response.results] == [ + result.model_dump() for result in expected + ] @pytest.mark.parametrize( @@ -177,7 +159,8 @@ def test_transform_search_response_attaches_only_the_exa_result_fields_it_return } ) - assert [result.model_extra for result in response.results] == [{}, expected_extras] + assert {key: response.results[1].model_extra[key] for key in expected_extras} == expected_extras + assert {key: response.results[1].model_extra[key] for key in exa_fields} == exa_fields @pytest.mark.parametrize( @@ -198,7 +181,7 @@ def test_transform_search_response_keeps_the_exa_envelope_as_provider_metadata( ): response: Final = _transform(payload) - assert response.model_extra == {"provider_metadata": expected_metadata} + assert response.model_extra["provider_metadata"] == expected_metadata assert response.model_dump(mode="json")["provider_metadata"] == expected_metadata @@ -283,3 +266,313 @@ def test_get_complete_url_ignores_a_trailing_slash_on_api_base(api_base: str) -> ExaAISearchConfig().get_complete_url(api_base=api_base, optional_params={}) == "https://gateway.example.com/exa/search" ) + + +OUTPUT_SCHEMA: Final = {"type": "object", "properties": {"answer": {"type": "string"}}} +OUTPUT: Final = { + "content": {"answer": "Grounded answer"}, + "grounding": [ + {"field": "answer", "citations": [{"url": "https://example.com", "title": "Source"}], "confidence": "high"} + ], +} +FULL_RESULT: Final = { + "title": "Source", + "url": "https://example.com", + "publishedDate": "2026-01-01", + "author": None, + "id": "document-id", + "image": "https://example.com/image.png", + "favicon": "https://example.com/icon.png", + "text": "", + "highlights": ["relevant passage", "another passage"], + "highlightScores": [0.8, 0.7], + "summary": "summary", + "subpages": [{"title": "Subpage", "url": "https://example.com/sub"}], + "extras": { + "links": ["https://example.com/link"], + "richLinks": [{"url": "https://example.com/link", "anchor": "Link"}], + }, + "entities": [{"id": "entity", "type": "company", "properties": {"name": "Example"}}], + "futureField": {"nested": [False, 0, None]}, +} +FULL_RESPONSE: Final = { + "results": [FULL_RESULT], + "output": OUTPUT, + "requestId": "request-id", + "costDollars": {"total": 0.012}, + "searchTime": 24, + "resolvedSearchType": "auto", + "context": "", + "futureMetadata": [], +} + + +@pytest.mark.parametrize("mode", ["instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.usefixtures("exa_api_key") +@pytest.mark.asyncio +async def test_public_search_preserves_native_contract(respx_mock: respx.MockRouter, mode: str, asynchronous: bool): + native: Final = { + "type": mode, + "additionalQueries": ["related question"], + "outputSchema": OUTPUT_SCHEMA, + "systemPrompt": "Use grounded sources", + "objective": "Research", + "userIdentifier": "opaque-id", + "userLocation": {"latitude": 40.7, "longitude": -74.0, "country": "US"}, + "includeDomains": ["example.com"], + "excludeDomains": ["other.example"], + "startPublishedDate": "2026-01-01T00:00:00Z", + "endPublishedDate": "2026-02-01T00:00:00Z", + "moderation": False, + "numResults": 3, + "category": "publication", + "compliance": "hipaa", + "contents": { + "highlights": {"query": "evidence", "dynamic": True, "verbosity": "medium"}, + "extras": {"links": 2, "imageLinks": 2, "richLinks": 2, "richImageLinks": 2, "codeBlocks": 2}, + "maxAgeHours": -1, + "livecrawlTimeout": 5000, + "subpages": 1, + "subpageTarget": ["about"], + }, + } + route: Final = respx_mock.post(EXA_SEARCH_URL).respond(json=FULL_RESPONSE) + headers: Final = {"Exa-Beta": "dynamic-highlights-2026-08-28"} + result: Final = ( + await litellm.asearch(query="question", search_provider="exa_ai", extra_headers=headers, **native) + if asynchronous + else litellm.search(query="question", search_provider="exa_ai", extra_headers=headers, **native) + ) + assert json.loads(route.calls.last.request.content) == {"query": "question", **native} + assert route.calls.last.request.headers["exa-beta"] == headers["Exa-Beta"] + dumped: Final = result.model_dump() + for key, value in FULL_RESPONSE.items(): + if key != "results": + assert dumped[key] == value + for key, value in FULL_RESULT.items(): + assert dumped["results"][0][key] == value + assert dumped["results"][0]["snippet"] == "\n\n".join(FULL_RESULT["highlights"]) + assert dumped["results"][0]["date"] == FULL_RESULT["publishedDate"] + assert json.loads(result.model_dump_json()) == dumped + + +@pytest.mark.parametrize( + "contents", + [ + None, + {}, + {"text": False}, + {"highlights": True}, + { + "text": { + "maxCharacters": 800, + "includeHtmlTags": True, + "verbosity": "full", + "includeSections": ["body"], + "excludeSections": ["footer"], + } + }, + {"summary": {"query": "summarize", "schema": OUTPUT_SCHEMA}}, + {"highlights": {"maxCharacters": 700, "query": "evidence"}, "snapshotAsOf": "2026-01-01"}, + {"text": True, "livecrawl": "never", "crawledBeforeDate": "2026-01-01", "context": {"maxCharacters": 200}}, + ], +) +@pytest.mark.usefixtures("exa_api_key") +def test_contents_are_sent_without_injecting_text(respx_mock: respx.MockRouter, contents: object): + route: Final = respx_mock.post(EXA_SEARCH_URL).respond(json={"results": []}) + litellm.search(query="question", search_provider="exa_ai", contents=contents) + assert json.loads(route.calls.last.request.content) == {"query": "question", "contents": contents} + + +@pytest.mark.parametrize( + "output", [None, {}, {"content": "", "grounding": []}, {"content": {}, "grounding": []}, OUTPUT] +) +def test_output_preserves_present_empty_and_null_values(output: object): + response: Final = _transform({"results": [], "output": output}) + assert response.model_dump()["output"] == output + + +def test_absent_output_stays_absent(): + assert _transform({"results": []}).model_dump() == {"results": [], "object": "search", "provider_metadata": {}} + + +@pytest.fixture(autouse=True) +def use_httpx_for_respx(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +def test_legacy_parameters_and_unified_precedence_are_preserved(): + legacy: Final = { + "max_results": 4, + "numResults": 9, + "country": "US", + "userLocation": "GB", + "search_domain_filter": ["example.com"], + "includeDomains": ["other.example"], + "startCrawlDate": "2020-01-01", + "endCrawlDate": "2021-01-01", + "includeText": ["one"], + "excludeText": ["two"], + "context": True, + "max_tokens_per_page": 100, + } + before: Final = json.dumps(legacy, sort_keys=True) + request: Final = ExaAISearchConfig().transform_search_request(["first", "second"], legacy) + assert request == { + "query": "first second", + "numResults": 4, + "userLocation": "US", + "includeDomains": ["example.com"], + "startCrawlDate": "2020-01-01", + "endCrawlDate": "2021-01-01", + "includeText": ["one"], + "excludeText": ["two"], + "context": True, + "contents": {"text": True}, + } + assert json.dumps(legacy, sort_keys=True) == before + + +@pytest.mark.parametrize("total", [0, 0.012, 2]) +@pytest.mark.parametrize("serialized", [False, True]) +def test_exa_reported_total_prices_the_whole_request(total: float, serialized: bool) -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + response: Final = SearchResponse.model_validate({"results": [], "costDollars": {"total": total}}) + payload: Final = response.model_dump() if serialized else response + assert search_provider_cost_per_query("exa_ai/search", "exa_ai", number_of_queries=3, response=payload) == ( + total, + 0.0, + ) + assert ( + litellm.completion_cost( + completion_response=payload, + model="exa_ai/search", + custom_llm_provider="exa_ai", + call_type="search", + optional_params={"query": ["one", "two"], "costDollars": {"total": 999}}, + ) + == total + ) + assert litellm.cost_per_token( + model="exa_ai/search", custom_llm_provider="exa_ai", call_type="search", response=payload + ) == (total, 0.0) + + +@pytest.mark.parametrize( + "reported", + [ + None, + {}, + {"total": None}, + {"total": -1}, + {"total": True}, + {"total": "0.01"}, + {"total": float("nan")}, + {"total": float("inf")}, + [], + ], +) +def test_exa_missing_or_invalid_total_uses_estimate(reported: object) -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + estimate: Final = search_provider_cost_per_query("exa_ai/search", "exa_ai") + assert ( + search_provider_cost_per_query( + "exa_ai/search", + "exa_ai", + response={"costDollars": reported}, + optional_params={"costDollars": {"total": 999}}, + ) + == estimate + ) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_exa_estimate_adds_each_result_above_the_included_ten() -> None: + from litellm.llms.exa_ai.search.cost_calculator import EXA_AI_ADDITIONAL_RESULT_COST + from litellm.search.cost_calculator import search_provider_cost_per_query + + base, _ = search_provider_cost_per_query("exa_ai/search", "exa_ai") + + assert base > 0.0 + assert search_provider_cost_per_query("exa_ai/search", "exa_ai", optional_params={"max_results": 5}) == (base, 0.0) + assert search_provider_cost_per_query( + "exa_ai/search", "exa_ai", optional_params={"max_results": 25} + ) == pytest.approx((base + 15 * EXA_AI_ADDITIONAL_RESULT_COST, 0.0)) + assert search_provider_cost_per_query( + "exa_ai/search", "exa_ai", optional_params={"numResults": 12} + ) == pytest.approx((base + 2 * EXA_AI_ADDITIONAL_RESULT_COST, 0.0)) + + +@pytest.mark.usefixtures("local_model_cost_map") +def test_exa_estimate_prices_one_request_however_many_queries_it_joins() -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + assert search_provider_cost_per_query( + "exa_ai/search", "exa_ai", number_of_queries=3 + ) == search_provider_cost_per_query("exa_ai/search", "exa_ai") + + +@pytest.mark.usefixtures("local_model_cost_map") +@pytest.mark.parametrize( + ("search_type", "pricing_model"), + [ + ("lite", "exa_ai/search-lite"), + ("instant", "exa_ai/search-instant"), + ("auto", "exa_ai/search"), + ("fast", "exa_ai/search"), + ("deep-lite", "exa_ai/search-deep"), + ("deep", "exa_ai/search-deep"), + ("deep-reasoning", "exa_ai/search-deep-reasoning"), + ], +) +def test_exa_estimate_uses_the_price_of_its_search_type(search_type: str, pricing_model: str) -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + model_info: Final = litellm.get_model_info(model=pricing_model, custom_llm_provider="exa_ai") + + assert search_provider_cost_per_query("exa_ai/search", "exa_ai", optional_params={"type": search_type}) == ( + model_info["input_cost_per_query"], + 0.0, + ) + + +@pytest.mark.parametrize("native_count", [1, 100]) +def test_exa_estimate_uses_the_result_count_sent_to_provider(native_count: int) -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + params: Final = {"max_results": 25, "numResults": native_count} + outgoing: Final = ExaAISearchConfig().transform_search_request("question", params) + assert outgoing["numResults"] == 25 + assert search_provider_cost_per_query("exa_ai/search", "exa_ai", optional_params=params) == ( + search_provider_cost_per_query("exa_ai/search", "exa_ai", optional_params=outgoing) + ) + + +@pytest.mark.parametrize("count", [True, -1, 1.5, float("nan"), float("inf"), "25"]) +def test_exa_estimate_ignores_invalid_result_counts(count: object) -> None: + from litellm.search.cost_calculator import search_provider_cost_per_query + + assert search_provider_cost_per_query( + "exa_ai/search", "exa_ai", optional_params={"numResults": count} + ) == search_provider_cost_per_query("exa_ai/search", "exa_ai") + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.usefixtures("exa_api_key") +@pytest.mark.asyncio +async def test_lite_search_preserves_mode_and_highlights(respx_mock: respx.MockRouter, asynchronous: bool) -> None: + route: Final = respx_mock.post(EXA_SEARCH_URL).respond(json=FULL_RESPONSE) + response: Final = ( + await litellm.asearch(query="question", search_provider="exa_ai", type="lite", contents={"highlights": True}) + if asynchronous + else litellm.search(query="question", search_provider="exa_ai", type="lite", contents={"highlights": True}) + ) + assert json.loads(route.calls.last.request.content) == { + "query": "question", + "type": "lite", + "contents": {"highlights": True}, + } + assert response.results[0].snippet == "\n\n".join(FULL_RESULT["highlights"])