From a955a0ef2a01f7355d9bb3e547418f4d950de657 Mon Sep 17 00:00:00 2001 From: kesku Date: Thu, 15 Oct 2026 00:26:49 -0400 Subject: [PATCH] chore(exa): keep live validation tooling local --- cookbook/exa_search/README.md | 148 ------------------ cookbook/exa_search/validate_live.py | 219 --------------------------- 2 files changed, 367 deletions(-) delete mode 100644 cookbook/exa_search/README.md delete mode 100644 cookbook/exa_search/validate_live.py diff --git a/cookbook/exa_search/README.md b/cookbook/exa_search/README.md deleted file mode 100644 index 596071de2bf..00000000000 --- a/cookbook/exa_search/README.md +++ /dev/null @@ -1,148 +0,0 @@ -# Exa search - -Use `litellm.search()` or `await litellm.asearch()` with `search_provider="exa_ai"` and `EXA_API_KEY`. Native Exa parameters retain their camelCase names. See the [current search contract](https://exa.ai/docs/reference/search). - -```python -import litellm - -response = litellm.search( - query="recent advances in battery recycling", - search_provider="exa_ai", - contents={"highlights": True}, -) -for result in response.results: - print(result.title, result.url, result.snippet) - print(result.model_dump().get("highlights")) -print(response.model_dump().get("requestId")) -``` - -Exa defaults to `type="auto"` and 10 results. The public `numResults` range is 1 to 100, with limits that may vary by mode. The current modes are `instant`, `fast`, `auto`, `deep-lite`, `deep`, and `deep-reasoning`. `additionalQueries` is for deep modes. LiteLLM forwards these values without translating them into legacy `neural` or `keyword` modes. Category hints, domain filters, publication windows, moderation, `objective`, `systemPrompt`, `userIdentifier`, and enterprise `compliance` are forwarded. Exa validates provider-specific limits and unsupported combinations. - -Exa defaults `moderation` and `stream` to false, text rendering verbosity to `compact`, `includeHtmlTags` to false, extras counts and `subpages` to zero, and `livecrawlTimeout` to 10,000 milliseconds. Omitted freshness controls use cached content when available and fetch pages without cached content. LiteLLM leaves these defaults to Exa. - -LiteLLM retains its existing compatibility default of `contents={"text": True}` when `contents` is omitted. Pass `contents={}` or `None` for metadata only, or request highlights explicitly as above. An explicit contents object is never supplemented with full text. `max_results`, `search_domain_filter`, and `country` map to `numResults`, `includeDomains`, and `userLocation`. When both spellings are supplied, the unified parameter takes precedence, as before. `max_tokens_per_page` is not mapped to a character limit. - -For location, `userLocation` also accepts `{"country": "US"}` or `{"latitude": 40.7, "longitude": -74.0}` with an optional country. Coordinates must be provided together. - -## Extraction options - -`contents.text` accepts a boolean or an object with `maxCharacters`, `includeHtmlTags`, `verbosity` (`compact`, `standard`, `full`), `includeSections`, and `excludeSections`. Section names are `header`, `navigation`, `banner`, `body`, `sidebar`, `footer`, and `metadata`. Section selection is best effort. Exa recommends `maxAgeHours=0` to apply rendering controls to freshly fetched content. - -`contents.highlights` accepts a boolean or an object with a selection `query`, `maxCharacters`, or the beta options below. `contents.summary` accepts an object with `query` and a JSON `schema`. Prefer one extraction mode appropriate to the task. Summary generation adds per-result synthesis. - -Freshness and discovery options are `maxAgeHours`, `livecrawlTimeout`, `snapshotAsOf`, `subpages`, and `subpageTarget`. `maxAgeHours` controls cache freshness, not publication recency. `snapshotAsOf` retrieves stored historical content and must not be combined with `maxAgeHours`. - -`contents.extras` accepts counts for `links`, `imageLinks`, `richLinks`, `richImageLinks`, and `codeBlocks`. Returned extras, subpages, entities, and unknown future fields are preserved. - -## Dynamic Highlights (research preview) - -The documented contract requires Exa's beta header for both `dynamic` and highlight `verbosity`. LiteLLM forwards caller-supplied `extra_headers`, including this header. It does not silently enroll requests in a beta. - -```python -response = await litellm.asearch( - query="recent advances in battery recycling", - search_provider="exa_ai", - contents={"highlights": {"dynamic": True, "verbosity": "medium"}}, - extra_headers={"Exa-Beta": "dynamic-highlights-2026-08-28"}, -) -``` - -Dynamic Highlights allocates a shared context budget across results. `verbosity` accepts `low`, `medium`, or `high`. Do not combine `dynamic` with `maxCharacters`, or `verbosity` with `maxCharacters` or `numSentences`. Access and behavior remain subject to Exa's preview availability. - -## Web-search interception - -To use Dynamic Highlights in the [proxy interception flow](https://docs.litellm.ai/docs/integrations/websearch_interception), configure the registered search tool. Add this alongside your existing `model_list`. - -```yaml -search_tools: - - search_tool_name: exa-dynamic - litellm_params: - search_provider: exa_ai - api_key: os.environ/EXA_API_KEY - contents: - highlights: - dynamic: true - verbosity: low - extra_headers: - Exa-Beta: dynamic-highlights-2026-08-28 -litellm_settings: - callbacks: [websearch_interception] - websearch_interception_params: - enabled_providers: [openai] - search_tool_name: exa-dynamic -``` - -The interceptor supplies the returned highlights as search snippets to the model, without requesting full text. Search costs are logged separately from model token costs. - -## Structured output - -```python -response = litellm.search( - query="recent advances in battery recycling", - search_provider="exa_ai", - contents={"highlights": True}, - systemPrompt="Use cited evidence. State when evidence is missing.", - outputSchema={ - "type": "object", - "properties": {"answer": {"type": "string"}}, - "required": ["answer"], - }, -) -output = response.model_dump()["output"] -print(output["content"]) -print(output["grounding"]) -``` - -`outputSchema` works across modes. Its root is `text` or `object`. Exa limits object schemas to 10 properties in total and two nesting levels, and arrays require `items`. Complete output and grounding are preserved without conversion. Missing output remains absent; present null or empty values remain present. - -## Streaming - -```python -stream = await litellm.asearch( - query="recent advances in battery recycling", - search_provider="exa_ai", - contents={"highlights": True}, - outputSchema={"type": "text", "description": "A short cited answer"}, - stream=True, -) -async with stream: - async for event in stream: - print(event.model_dump()) -print(stream.final_response.model_dump()) -``` - -The synchronous equivalent uses `with litellm.search(..., stream=True) as stream` and `for event in stream`. Use the context manager or call `close()` / `await aclose()` when abandoning a stream early. Early close and cancellation record LiteLLM's per-query charge once, using the results received so far; they do not populate `final_response` without a `done` event. - -Events preserve their native `type` and payload. The documented types are `text-delta`, `grounding`, `results`, `stream-reset`, `done`, and `error`. On `stream-reset`, discard previously accumulated synthesized text. `done.output` is authoritative. `final_response` becomes available after `done`, combining the most recent results with the terminal output and metadata. Provider error events, malformed events, and premature disconnects raise an exception and close the response. Heartbeats are ignored. SSE framing is decoded incrementally, including fragmented UTF-8 and multiline data. - -Exa sends ordinary JSON for `stream=True` without an `outputSchema`. In that case the SDK returns a regular `SearchResponse`, not an iterator. The HTTP content type determines how the response is decoded, including when a server returns JSON despite an output schema. `complete_response=True` consumes an SSE stream and returns its final `SearchResponse`; a JSON response is returned directly. Completed streams use the same success callbacks and cost logging as other completed LiteLLM calls. - -## Response compatibility - -`response.results` retains LiteLLM's `title`, `url`, `snippet`, `date`, and `last_updated`. Snippets prefer nonempty text, then joined highlights, then summary. Every original Exa result field is also preserved, including text, highlights, scores, author, publication date, images, extras, subpages, and entities. Top-level `requestId`, `output`, `costDollars`, `searchTime`, legacy fields, and unknown extensions survive `model_dump()` and `model_dump_json()`. - -`costDollars` is Exa's estimate, not an invoice. LiteLLM uses a finite, nonnegative `costDollars.total` as the base search cost, including zero, for JSON and completed SSE responses. An explicit `input_cost_per_query` overrides the provider total. Existing discounts and billing policies still apply. If the total is missing or invalid, or a stream closes before the final total arrives, LiteLLM falls back to its configured per-query estimate. That fallback may differ from Exa's final charge. - -## Deprecated fields - -Legacy callers can still forward `startCrawlDate`, `endCrawlDate`, `includeText`, `excludeText`, and `context`. Exa documents crawl-date filters as ignored. Prefer publication windows for publication dates and `contents.snapshotAsOf` for stored historical content. Use text or highlights instead of context. - -Nested legacy fields `contents.livecrawl`, `contents.crawledBeforeDate`, and highlight `numSentences` / `highlightsPerUrl` also pass through for compatibility. Prefer `maxAgeHours`, `snapshotAsOf`, and highlights defaults or `maxCharacters`, respectively. LiteLLM does not reinterpret deprecated fields or silently delete them. - -## Validation - -Run the offline adapter, fragmented-SSE, and real local HTTP tests with the repository's locked development environment. - -```bash -uv run --no-sync pytest tests/unit/llms/exa_ai/search \ - tests/unit/llms/base_llm/search/test_streaming_iterator.py \ - tests/integration/sdk/test_exa_search_wire.py -``` - -To make paid live requests using an already configured `EXA_API_KEY`, run the opt-in matrix. It writes returned data and event sequences, but not credentials. It alternates synchronous and asynchronous calls and checks object output against its JSON schema. - -```bash -uv run --no-sync python cookbook/exa_search/validate_live.py --output /tmp/exa-validation.json -``` - -Use `--case dynamic-highlights` or another case name to run a subset. Preview access, enterprise compliance, source availability, and best-effort extraction can vary between accounts and requests. Acceptance of a subpage or extras request does not guarantee those fields will be present for every source. diff --git a/cookbook/exa_search/validate_live.py b/cookbook/exa_search/validate_live.py deleted file mode 100644 index 887a8a4a0b2..00000000000 --- a/cookbook/exa_search/validate_live.py +++ /dev/null @@ -1,219 +0,0 @@ -"""Run explicitly opted-in Exa checks, writing responses without credentials.""" - -import argparse -import asyncio -import json -import os -import time -from pathlib import Path -from typing import Final -from urllib.parse import urlsplit - -import jsonschema - -import litellm -from litellm.llms.base_llm.search.streaming_iterator import SearchStreamWrapper - -QUERY: Final = "What does the Python documentation say about asyncio TaskGroup?" -SCHEMA: Final = {"type": "object", "properties": {"answer": {"type": "string"}}, "required": ["answer"]} - - -def cases(): - for mode in ("instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"): - yield f"{mode}-highlights", {"type": mode, "contents": {"highlights": True}} - yield ( - "dynamic-highlights", - { - "contents": {"highlights": {"dynamic": True, "verbosity": "low"}}, - "extra_headers": {"Exa-Beta": "dynamic-highlights-2026-08-28"}, - }, - ) - yield ( - "text-options", - { - "contents": { - "text": {"maxCharacters": 1200, "verbosity": "compact", "includeSections": ["body"]}, - "maxAgeHours": 0, - } - }, - ) - yield "summary-options", {"contents": {"summary": {"query": "Explain TaskGroup"}}} - yield "metadata-only", {"contents": {}} - yield ( - "object-output", - {"contents": {"highlights": True}, "outputSchema": SCHEMA, "systemPrompt": "Ground the answer in the sources."}, - ) - yield "text-output", {"contents": {"highlights": True}, "outputSchema": {"type": "text"}} - yield ( - "deep-additional-queries", - { - "type": "deep", - "additionalQueries": ["Python TaskGroup exception handling"], - "contents": {"highlights": True}, - }, - ) - yield "location-object", {"userLocation": {"country": "US"}, "contents": {"highlights": True}} - yield ( - "extras", - { - "contents": { - "text": {"maxCharacters": 500}, - "extras": {"links": 2, "richLinks": 2, "imageLinks": 2, "richImageLinks": 2, "codeBlocks": 2}, - } - }, - ) - yield "snapshot", {"contents": {"highlights": True, "snapshotAsOf": "2026-01-01"}} - yield "stream-text", {"stream": True, "outputSchema": {"type": "text"}, "contents": {"highlights": True}} - yield "stream-object", {"stream": True, "outputSchema": SCHEMA, "contents": {"highlights": True}} - yield "stream-json-fallback", {"stream": True, "contents": {"highlights": True}} - yield "legacy-default-text", {} - yield "null-contents", {"contents": None} - yield "highlight-options", {"contents": {"highlights": {"query": "TaskGroup exceptions", "maxCharacters": 600}}} - yield "summary-schema", {"contents": {"summary": {"query": "Explain TaskGroup", "schema": SCHEMA}}} - yield "location-coordinates", {"userLocation": {"latitude": 40.7, "longitude": -74.0}, "contents": {}} - yield "domain-filter", {"includeDomains": ["docs.python.org"], "contents": {"highlights": True}} - yield "subpages", {"contents": {"highlights": True, "subpages": 1, "subpageTarget": ["asyncio"]}} - yield ( - "text-html-sections", - { - "contents": { - "text": {"includeHtmlTags": True, "excludeSections": ["navigation", "footer"], "maxCharacters": 600}, - "maxAgeHours": 0, - } - }, - ) - - for mode in ("instant", "fast", "auto", "deep-lite", "deep", "deep-reasoning"): - yield ( - f"{mode}-stream-object", - {"type": mode, "stream": True, "outputSchema": SCHEMA, "contents": {"highlights": True}}, - ) - - yield ( - "subpages-homepage", - { - "query": "Python official documentation homepage", - "includeDomains": ["docs.python.org"], - "contents": {"text": {"maxCharacters": 500}, "subpages": 2, "subpageTarget": ["library"]}, - }, - ) - yield ( - "image-extras", - { - "query": "Wikimedia Commons cat photographs", - "includeDomains": ["commons.wikimedia.org"], - "contents": {"text": {"maxCharacters": 500}, "extras": {"imageLinks": 2, "richImageLinks": 2}}, - }, - ) - yield ( - "native-controls", - { - "query": "Research papers on structured concurrency", - "type": "fast", - "category": "publication", - "moderation": True, - "startPublishedDate": "2000-01-01T00:00:00Z", - "endPublishedDate": "2027-01-01T00:00:00Z", - "excludeDomains": ["example.com"], - "objective": "Find research evidence on structured concurrency", - "userIdentifier": "litellm-validation", - "contents": {"highlights": True, "maxAgeHours": -1, "livecrawlTimeout": 5000}, - }, - ) - - -async def run(asynchronous: bool, parameters): - response = await litellm.asearch(**parameters) if asynchronous else litellm.search(**parameters) - if isinstance(response, SearchStreamWrapper): - if asynchronous: - async with response: - events = [event.model_dump() async for event in response] - else: - with response: - events = [event.model_dump() for event in response] - assert events[-1]["type"] == "done" - assert response.final_response is not None - return response.final_response.model_dump(), events - return response.model_dump(), [] - - -async def main(output: Path, selected: list[str]): - key = os.environ.get("EXA_API_KEY") - if not key: - raise SystemExit("EXA_API_KEY is unavailable. No live requests were made.") - output.parent.mkdir(parents=True, exist_ok=True) - records = [] - for index, (name, params) in enumerate(cases()): - if selected and name not in selected: - continue - started = time.monotonic() - asynchronous = index % 2 == 0 - try: - response, events = await run( - asynchronous, {"query": QUERY, "search_provider": "exa_ai", "numResults": 2, "timeout": 120, **params} - ) - if params.get("stream") and params.get("outputSchema") is not None: - assert events, "Requested SSE but received ordinary JSON" - assert isinstance(response["results"], list) - if "requestId" in response: - assert isinstance(response["requestId"], str) - if "searchTime" in response: - assert isinstance(response["searchTime"], (int, float)) - if "costDollars" in response: - assert isinstance(response["costDollars"], dict) - if params.get("outputSchema") == SCHEMA: - jsonschema.validate(response["output"]["content"], SCHEMA) - assert isinstance(response["output"]["grounding"], list) - if name.endswith("highlights"): - assert any(result.get("highlights") for result in response["results"]) - assert all(not result.get("text") for result in response["results"]) - if name in ("text-options", "legacy-default-text", "text-html-sections"): - assert any(result.get("text") for result in response["results"]) - if name in ("summary-options", "summary-schema"): - assert any(result.get("summary") for result in response["results"]) - if name == "summary-schema": - for result in response["results"]: - if result.get("summary"): - jsonschema.validate(json.loads(result["summary"]), SCHEMA) - if params.get("outputSchema", {}).get("type") == "text": - assert isinstance(response["output"]["content"], str) - assert isinstance(response["output"]["grounding"], list) - if name == "domain-filter": - assert response["results"] - assert all(urlsplit(result["url"]).hostname == "docs.python.org" for result in response["results"]) - if name in ("null-contents", "metadata-only"): - assert all(not result.get("text") for result in response["results"]) - records.append( - { - "case": name, - "async": asynchronous, - "parameters": params, - "status": "passed", - "seconds": time.monotonic() - started, - "response": response, - "events": events, - } - ) - except Exception as error: - records.append( - { - "case": name, - "async": asynchronous, - "parameters": params, - "status": "failed", - "seconds": time.monotonic() - started, - "error": repr(error).replace(key, "[REDACTED]"), - } - ) - output.write_text(json.dumps(records, indent=2)) - print(name, records[-1]["status"], flush=True) - if any(record["status"] != "passed" for record in records): - raise SystemExit(1) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--case", action="append", default=[]) - args = parser.parse_args() - asyncio.run(main(args.output, args.case))