From d3599a0729db3298debe8fb244d0eedfb47c7aa4 Mon Sep 17 00:00:00 2001 From: Deepanshu Date: Sat, 5 Sep 2026 09:14:00 -0400 Subject: [PATCH 1/3] fix(exa): populate highlights, highlight_scores, and score in search results ExaAISearchConfig.transform_search_response only read the "text" field from each Exa result, so a request using contents.highlights or contents.summary got back an empty snippet even though Exa returned and billed for that content. score (returned for type: neural searches) and highlightScores were dropped the same way. snippet now falls back from text to highlights (joined) to summary, and highlights, highlight_scores, summary, and score are attached to each SearchResult when Exa actually returns them, using the model's existing extra="allow" config. Malformed provider responses (explicit nulls, wrong field types, non-dict entries) degrade gracefully instead of crashing. --- litellm/llms/exa_ai/search/transformation.py | 105 +++++- .../llms/exa_ai/search/test_transformation.py | 350 ++++++++++++++++++ 2 files changed, 441 insertions(+), 14 deletions(-) create mode 100644 tests/test_litellm/llms/exa_ai/search/test_transformation.py diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 022622d7af7..0969c4a6e4b 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -4,9 +4,12 @@ Calls Exa AI's /search endpoint to search the web. Exa AI API Reference: https://docs.exa.ai/reference/search """ +from collections.abc import Mapping +from types import MappingProxyType from typing import Final, TypedDict import httpx +from pydantic import BaseModel, ConfigDict from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.search.transformation import ( @@ -46,6 +49,89 @@ class ExaAISearchRequest(_ExaAISearchRequestRequired, total=False): contents: dict # Optional - content retrieval options +_HIGHLIGHT_SEPARATOR: Final[str] = "\n\n" + +_NOTHING: Final[Mapping[str, object]] = MappingProxyType({}) + + +def _optional(key: str, value: object) -> Mapping[str, object]: + """A one-entry mapping to spread into a SearchResult, or nothing when the value is absent.""" + return MappingProxyType({key: value}) if value is not None else _NOTHING + + +class _ExaHighlightFields(BaseModel): + """ + Parses just the two content-mode fields whose runtime shape Exa doesn't guarantee, + typed as `object` (not the documented `list[str]`/`list[float]` shape) so + `_exa_highlights`/`_exa_highlight_scores` can isinstance-check them meaningfully + against a real unknown, rather than a shape basedpyright would otherwise trust. + """ + + model_config = ConfigDict(extra="ignore", frozen=True) + + highlights: object = None + highlightScores: object = None + + +def _exa_highlights(result: Mapping[str, object]) -> tuple[object, ...] | None: + """ + Exa documents `highlights` as a list of strings, but a malformed response (e.g. a bare + string) must not be silently iterated character-by-character. Items are passed through + as-is rather than filtered by type, since `highlightScores[i]` is Exa's relevance score + for `highlights[i]`; independently filtering either array by item type would desync + that positional pairing. + """ + raw: Final = _ExaHighlightFields.model_validate(result).highlights + return tuple(raw) if isinstance(raw, (list, tuple)) else None + + +def _exa_highlight_scores(result: Mapping[str, object]) -> tuple[object, ...] | None: + raw: Final = _ExaHighlightFields.model_validate(result).highlightScores + return tuple(raw) if isinstance(raw, (list, tuple)) else None + + +def _as_str(value: object) -> str | None: + return value if isinstance(value, str) else None + + +def _exa_snippet(result: Mapping[str, object]) -> str: + """ + Exa returns each requested content mode in its own field, and omits `text` + entirely when only `highlights` or `summary` were asked for. Non-string highlight + entries are dropped only for this joined-snippet computation, not from the raw + `highlights` extra field, and blank entries are dropped too, since joining only + blanks would otherwise produce a non-empty separator-only string that wrongly wins + over `summary`. + """ + highlights_snippet: Final = _HIGHLIGHT_SEPARATOR.join( + h for h in (_exa_highlights(result) or ()) if isinstance(h, str) and h.strip() + ) + return _as_str(result.get("text")) or highlights_snippet or _as_str(result.get("summary")) or "" + + +def _exa_results(response_json: object) -> tuple[Mapping[str, object], ...]: + """Filters out a malformed `results` entry (e.g. `null`) instead of letting it + crash `.get()` calls downstream.""" + raw: Final = response_json.get("results") if isinstance(response_json, dict) else None + if not isinstance(raw, (list, tuple)): + return () + return tuple(r for r in raw if isinstance(r, dict)) + + +def _to_search_result(result: Mapping[str, object]) -> SearchResult: + return SearchResult( + title=_as_str(result.get("title")) or "", + url=_as_str(result.get("url")) or "", + snippet=_exa_snippet(result), + date=_as_str(result.get("publishedDate")), # ISO 8601 datetime string + last_updated=None, # Exa AI doesn't provide last_updated in response + **_optional("highlights", _exa_highlights(result)), + **_optional("highlight_scores", _exa_highlight_scores(result)), + **_optional("summary", result.get("summary")), + **_optional("score", result.get("score")), + ) + + class ExaAISearchConfig(BaseSearchConfig): EXA_AI_API_BASE = "https://api.exa.ai" @@ -164,7 +250,8 @@ class ExaAISearchConfig(BaseSearchConfig): Exa AI → LiteLLM mappings: - results[].title → SearchResult.title - results[].url → SearchResult.url - - results[].text → SearchResult.snippet + - results[].text, else results[].highlights, else results[].summary → SearchResult.snippet + - results[].highlights, results[].highlightScores, results[].summary, results[].score → passed through when present - results[].publishedDate → SearchResult.date - No last_updated field in Exa AI response (set to None) @@ -177,19 +264,9 @@ class ExaAISearchConfig(BaseSearchConfig): """ response_json: Final = raw_response.json() - # Transform results to SearchResult objects - results: Final = [] - for result in response_json.get("results", []): - search_result = SearchResult( - title=result.get("title", ""), - url=result.get("url", ""), - snippet=result.get("text", ""), # Exa AI uses "text" for content - date=result.get("publishedDate"), # ISO 8601 datetime string - last_updated=None, # Exa AI doesn't provide last_updated in response - ) - results.append(search_result) - return SearchResponse( - results=results, + results=[ # mutable-ok: SearchResponse.results is declared list[SearchResult] + _to_search_result(result) for result in _exa_results(response_json) + ], object="search", ) diff --git a/tests/test_litellm/llms/exa_ai/search/test_transformation.py b/tests/test_litellm/llms/exa_ai/search/test_transformation.py new file mode 100644 index 00000000000..57ae4e0180c --- /dev/null +++ b/tests/test_litellm/llms/exa_ai/search/test_transformation.py @@ -0,0 +1,350 @@ +""" +Tests for Exa AI search response transformation. + +Regression coverage for https://github.com/BerriAI/litellm/issues/37502 and +https://github.com/BerriAI/litellm/issues/36905: Exa returns each requested +content mode (`text`, `highlights`, `summary`) in its own response field and +omits `text` entirely when only `highlights` or `summary` were requested, so +reading only `text` silently dropped billed content and returned an empty +snippet. `highlightScores` and `score` were dropped the same way. +""" + +import json +from unittest.mock import Mock + +import httpx +import pytest + +from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig + + +def _raw_response(payload: dict[str, object]) -> httpx.Response: + return httpx.Response( + status_code=200, + content=json.dumps(payload).encode(), + request=httpx.Request("POST", "https://api.exa.ai/search"), + ) + + +@pytest.fixture +def config() -> ExaAISearchConfig: + return ExaAISearchConfig() + + +class TestExaAISnippetFallback: + def test_text_only_produces_original_five_key_result(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "text": "full page text", + "publishedDate": "2026-01-01T00:00:00.000Z", + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.model_dump(exclude_none=True) == { + "title": "t", + "url": "https://example.com", + "snippet": "full page text", + "date": "2026-01-01T00:00:00.000Z", + } + + def test_highlights_fill_snippet_and_are_attached_when_text_absent(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "highlights": ["first highlight", "second highlight"], + "highlightScores": [0.9, 0.7], + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "first highlight\n\nsecond highlight" + assert result.highlights == ("first highlight", "second highlight") + assert result.highlight_scores == (0.9, 0.7) + + def test_summary_fills_snippet_when_text_and_highlights_absent(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "summary": "a short summary"}]}), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "a short summary" + assert result.summary == "a short summary" + + def test_text_wins_over_highlights_and_summary(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "text": "full page text", + "highlights": ["a highlight"], + "summary": "a summary", + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "full page text" + assert result.highlights == ("a highlight",) + assert result.summary == "a summary" + + def test_highlights_win_over_summary_for_snippet(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "highlights": ["a highlight"], "summary": "a summary"}]}), + logging_obj=Mock(), + ) + + assert response.results[0].snippet == "a highlight" + + def test_empty_highlights_list_falls_through_to_summary(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "highlights": [], "summary": "a summary"}]}), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "a summary" + assert result.highlights == () + + def test_highlights_of_only_empty_strings_falls_through_to_summary(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + {"results": [{"title": "t", "url": "https://example.com", "highlights": ["", ""], "summary": "a real summary"}]} + ), + logging_obj=Mock(), + ) + + assert response.results[0].snippet == "a real summary" + + def test_empty_string_when_no_content_fields_present(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com"}]}), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "" + assert result.model_dump(exclude_none=True) == { + "title": "t", + "url": "https://example.com", + "snippet": "", + } + + def test_explicit_null_highlights_and_highlight_scores_do_not_crash(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "text": "full page text", + "highlights": None, + "highlightScores": None, + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "full page text" + assert "highlights" not in result.model_dump(exclude_none=True) + assert "highlight_scores" not in result.model_dump(exclude_none=True) + + def test_highlights_as_a_bare_string_is_not_iterated_character_by_character( + self, config: ExaAISearchConfig + ) -> None: + """A malformed response sending `highlights` as a string, not a list, must not be + silently treated as an iterable of characters.""" + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "highlights": "abc"}]}), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "" + assert "highlights" not in result.model_dump(exclude_none=True) + + def test_highlights_list_with_non_string_items_does_not_crash(self, config: ExaAISearchConfig) -> None: + """Non-string items in `highlights` fall out of the joined snippet (since a snippet + must be a string) but are preserved as-is on the raw `highlights` field, not + dropped, so as not to desync `highlightScores`' positional correspondence.""" + response = config.transform_search_response( + raw_response=_raw_response( + {"results": [{"title": "t", "url": "https://example.com", "highlights": [1, 2, 3], "summary": "a summary"}]} + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "a summary" + assert result.highlights == (1, 2, 3) + + def test_highlights_and_highlight_scores_stay_positionally_paired_when_one_has_a_bad_item( + self, config: ExaAISearchConfig + ) -> None: + """highlightScores[i] is Exa's relevance score for highlights[i]; a malformed entry + in only one of the two arrays must not silently desync that pairing by dropping an + entry from one array but not the other.""" + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "highlights": ["a", 42, "b"], + "highlightScores": [0.9, 0.8, 0.7], + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.highlights == ("a", 42, "b") + assert result.highlight_scores == (0.9, 0.8, 0.7) + + def test_non_string_text_and_summary_do_not_crash(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "text": 12345}]}), + logging_obj=Mock(), + ) + assert response.results[0].snippet == "" + + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "summary": {"nested": "x"}}]}), + logging_obj=Mock(), + ) + assert response.results[0].snippet == "" + + def test_non_string_title_url_and_date_do_not_crash(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + {"results": [{"title": 123, "url": ["not", "a", "url"], "text": "hi", "publishedDate": 20260101}]} + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.title == "" + assert result.url == "" + assert result.date is None + assert result.snippet == "hi" + + def test_explicit_null_results_does_not_crash(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": None}), + logging_obj=Mock(), + ) + + assert response.results == [] + + def test_null_entry_within_results_list_does_not_crash(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [None, {"title": "t", "url": "https://example.com", "text": "hi"}]}), + logging_obj=Mock(), + ) + + assert len(response.results) == 1 + assert response.results[0].snippet == "hi" + + def test_whitespace_only_highlights_fall_through_to_summary(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + {"results": [{"title": "t", "url": "https://example.com", "highlights": [" ", "\t"], "summary": "a real summary"}]} + ), + logging_obj=Mock(), + ) + + assert response.results[0].snippet == "a real summary" + + +class TestExaAIScore: + def test_neural_search_score_is_attached(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "text": "full page text", + "score": 0.9438, + } + ] + } + ), + logging_obj=Mock(), + ) + + assert response.results[0].score == 0.9438 + + def test_score_absent_when_not_returned(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "text": "full page text"}]}), + logging_obj=Mock(), + ) + + assert "score" not in response.results[0].model_dump(exclude_none=True) + + def test_zero_score_is_attached_not_treated_as_absent(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response({"results": [{"title": "t", "url": "https://example.com", "text": "full page text", "score": 0.0}]}), + logging_obj=Mock(), + ) + + assert response.results[0].score == 0.0 + + def test_score_highlights_and_highlight_scores_all_attached_together(self, config: ExaAISearchConfig) -> None: + response = config.transform_search_response( + raw_response=_raw_response( + { + "results": [ + { + "title": "t", + "url": "https://example.com", + "highlights": ["a highlight", "another highlight"], + "highlightScores": [0.95, 0.42], + "score": 0.87, + "publishedDate": "2026-01-01T00:00:00.000Z", + } + ] + } + ), + logging_obj=Mock(), + ) + + result = response.results[0] + assert result.snippet == "a highlight\n\nanother highlight" + assert result.highlights == ("a highlight", "another highlight") + assert result.highlight_scores == (0.95, 0.42) + assert result.score == 0.87 + assert result.date == "2026-01-01T00:00:00.000Z" From ac05e4d1694f944ae43e2b09260c43e423994b49 Mon Sep 17 00:00:00 2001 From: Deepanshu Date: Sat, 5 Sep 2026 09:23:55 -0400 Subject: [PATCH 2/3] refactor(exa): parse highlight fields once per result, drop restated docstring Each result validated the same Exa fields three times via separate model_validate calls in _exa_highlights and _exa_highlight_scores; now parsed once in _to_search_result and threaded through. Also removes _optional's docstring, which only restated the one-line ternary below it. --- litellm/llms/exa_ai/search/transformation.py | 21 ++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 0969c4a6e4b..6f3ba57413c 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -55,7 +55,6 @@ _NOTHING: Final[Mapping[str, object]] = MappingProxyType({}) def _optional(key: str, value: object) -> Mapping[str, object]: - """A one-entry mapping to spread into a SearchResult, or nothing when the value is absent.""" return MappingProxyType({key: value}) if value is not None else _NOTHING @@ -73,7 +72,7 @@ class _ExaHighlightFields(BaseModel): highlightScores: object = None -def _exa_highlights(result: Mapping[str, object]) -> tuple[object, ...] | None: +def _exa_highlights(fields: _ExaHighlightFields) -> tuple[object, ...] | None: """ Exa documents `highlights` as a list of strings, but a malformed response (e.g. a bare string) must not be silently iterated character-by-character. Items are passed through @@ -81,12 +80,12 @@ def _exa_highlights(result: Mapping[str, object]) -> tuple[object, ...] | None: for `highlights[i]`; independently filtering either array by item type would desync that positional pairing. """ - raw: Final = _ExaHighlightFields.model_validate(result).highlights + raw: Final = fields.highlights return tuple(raw) if isinstance(raw, (list, tuple)) else None -def _exa_highlight_scores(result: Mapping[str, object]) -> tuple[object, ...] | None: - raw: Final = _ExaHighlightFields.model_validate(result).highlightScores +def _exa_highlight_scores(fields: _ExaHighlightFields) -> tuple[object, ...] | None: + raw: Final = fields.highlightScores return tuple(raw) if isinstance(raw, (list, tuple)) else None @@ -94,7 +93,7 @@ def _as_str(value: object) -> str | None: return value if isinstance(value, str) else None -def _exa_snippet(result: Mapping[str, object]) -> str: +def _exa_snippet(result: Mapping[str, object], highlights: tuple[object, ...] | None) -> str: """ Exa returns each requested content mode in its own field, and omits `text` entirely when only `highlights` or `summary` were asked for. Non-string highlight @@ -104,7 +103,7 @@ def _exa_snippet(result: Mapping[str, object]) -> str: over `summary`. """ highlights_snippet: Final = _HIGHLIGHT_SEPARATOR.join( - h for h in (_exa_highlights(result) or ()) if isinstance(h, str) and h.strip() + h for h in (highlights or ()) if isinstance(h, str) and h.strip() ) return _as_str(result.get("text")) or highlights_snippet or _as_str(result.get("summary")) or "" @@ -119,14 +118,16 @@ def _exa_results(response_json: object) -> tuple[Mapping[str, object], ...]: def _to_search_result(result: Mapping[str, object]) -> SearchResult: + fields: Final = _ExaHighlightFields.model_validate(result) + highlights: Final = _exa_highlights(fields) return SearchResult( title=_as_str(result.get("title")) or "", url=_as_str(result.get("url")) or "", - snippet=_exa_snippet(result), + snippet=_exa_snippet(result, highlights), date=_as_str(result.get("publishedDate")), # ISO 8601 datetime string last_updated=None, # Exa AI doesn't provide last_updated in response - **_optional("highlights", _exa_highlights(result)), - **_optional("highlight_scores", _exa_highlight_scores(result)), + **_optional("highlights", highlights), + **_optional("highlight_scores", _exa_highlight_scores(fields)), **_optional("summary", result.get("summary")), **_optional("score", result.get("score")), ) From 6111e927138852ea76b02c5d2f927b23ac0da934 Mon Sep 17 00:00:00 2001 From: Deepanshu Date: Sat, 5 Sep 2026 09:34:38 -0400 Subject: [PATCH 3/3] test(exa): rename test file to avoid basename collision with fastcrw's search test tests/test_litellm/llms/fastcrw/search/test_transformation.py already used this basename, and neither search/ directory has an __init__.py, so pytest's full-suite collection raised "import file mismatch" once both existed. --- ...est_transformation.py => test_exa_ai_search_transformation.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_litellm/llms/exa_ai/search/{test_transformation.py => test_exa_ai_search_transformation.py} (100%) diff --git a/tests/test_litellm/llms/exa_ai/search/test_transformation.py b/tests/test_litellm/llms/exa_ai/search/test_exa_ai_search_transformation.py similarity index 100% rename from tests/test_litellm/llms/exa_ai/search/test_transformation.py rename to tests/test_litellm/llms/exa_ai/search/test_exa_ai_search_transformation.py