diff --git a/tests/e2e/coverage_registry/llm_nonconversational.yaml b/tests/e2e/coverage_registry/llm_nonconversational.yaml index 0439cc5ed85..f581a82d29a 100644 --- a/tests/e2e/coverage_registry/llm_nonconversational.yaml +++ b/tests/e2e/coverage_registry/llm_nonconversational.yaml @@ -39,8 +39,6 @@ - {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets and /calls reachable with auth"} - {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"} - {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"} -- {id: llm.search.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: search, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.14 / LIT-4778", rationale: "POST /v1/search returns results for a registered tool"} -- {id: llm.search.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: search, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.14 / LIT-4778", rationale: "Search missing/empty/invalid query rejected"} - {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"} - {id: llm.bedrock_native.bedrock_converse.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse-stream"} - {id: llm.bedrock_native.bedrock_converse.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock converse missing/empty messages and invalid model"} diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py index a7032aeeac9..7146c01a1a3 100644 --- a/tests/e2e/coverage_registry/schema.py +++ b/tests/e2e/coverage_registry/schema.py @@ -40,7 +40,6 @@ LlmEndpoint = Literal[ "moderations", "realtime", "vector_stores", - "search", "ocr", "bedrock_native", ] diff --git a/tests/e2e/llm_translation/test_search_e2e.py b/tests/e2e/llm_translation/test_search_e2e.py deleted file mode 100644 index 9268a37a011..00000000000 --- a/tests/e2e/llm_translation/test_search_e2e.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Vendor §9.14: POST /v1/search through a registered search tool (LIT-4778). - -Registers a Perplexity-backed search tool at runtime, runs a basic search, and -pins missing/empty/invalid query handling. -""" - -from __future__ import annotations - -import os - -import pytest -from pydantic import BaseModel - -from e2e_config import unique_marker -from e2e_http import NoBody, unwrap, assert_client_error -from lifecycle import ResourceManager -from proxy_client import ProxyClient - -pytestmark = pytest.mark.e2e - - -class SearchToolLiteLLMParams(BaseModel): - search_provider: str - - -class SearchToolBody(BaseModel): - search_tool_name: str - litellm_params: SearchToolLiteLLMParams - search_tool_info: dict[str, str] | None = None - - -class CreateSearchToolRequest(BaseModel): - search_tool: SearchToolBody - - -class SearchToolResponse(BaseModel): - search_tool_id: str | None = None - search_tool_name: str | None = None - - -class SearchRequest(BaseModel): - search_tool_name: str | None = None - query: str | None = None - max_results: int | None = None - country: str | None = None - - -class SearchResultItem(BaseModel): - title: str | None = None - url: str | None = None - - -class SearchResponse(BaseModel): - object: str | None = None - results: list[SearchResultItem] = [] - - -def _search_provider() -> str: - if os.environ.get("PERPLEXITY_API_KEY") or os.environ.get("PERPLEXITYAI_API_KEY"): - return "perplexity" - if os.environ.get("TAVILY_API_KEY"): - return "tavily" - pytest.fail("set PERPLEXITY_API_KEY or TAVILY_API_KEY for /v1/search e2e coverage") - - -def _register_search_tool(proxy: ProxyClient, resources: ResourceManager) -> str: - provider = _search_provider() - name = f"e2e-search-{unique_marker()}" - created = unwrap( - proxy.transport.post( - "/search_tools", - headers=proxy.transport.master, - json=CreateSearchToolRequest( - search_tool=SearchToolBody( - search_tool_name=name, - litellm_params=SearchToolLiteLLMParams(search_provider=provider), - search_tool_info={"description": "e2e search tool"}, - ) - ), - response_type=SearchToolResponse, - ) - ) - tool_id = created.search_tool_id - if tool_id is not None: - def _delete_tool() -> None: - _ = proxy.transport.delete( - f"/search_tools/{tool_id}", - headers=proxy.transport.master, - json=NoBody(), - response_type=NoBody, - ) - - resources.defer(_delete_tool) - return name - - -class TestSearch: - @pytest.mark.covers("llm.search.openai.basic.nonstream.works") - def test_basic_search_returns_results( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: - tool = _register_search_tool(proxy, resources) - key = resources.key() - result = unwrap( - proxy.transport.post( - "/v1/search", - headers=proxy.transport.bearer(key), - json=SearchRequest( - search_tool_name=tool, - query="latest AI news", - max_results=3, - country="US", - ), - response_type=SearchResponse, - ) - ) - assert result.object in (None, "search") - assert result.results, f"search returned no results: {result}" - assert any(item.title or item.url for item in result.results), ( - f"search results lack title/url: {result}" - ) - - @pytest.mark.covers("llm.search.openai.basic.nonstream.works") - @pytest.mark.parametrize("max_results", [1, 5, 10]) - def test_max_results_boundaries( - self, proxy: ProxyClient, resources: ResourceManager, max_results: int - ) -> None: - tool = _register_search_tool(proxy, resources) - key = resources.key() - result = unwrap( - proxy.transport.post( - "/v1/search", - headers=proxy.transport.bearer(key), - json=SearchRequest( - search_tool_name=tool, query="weather forecast", max_results=max_results - ), - response_type=SearchResponse, - ) - ) - assert result.results, f"search returned no results for max_results={max_results}" - assert len(result.results) <= max_results, ( - f"expected <= {max_results} results, got {len(result.results)}" - ) - - @pytest.mark.covers("llm.search.openai.input_validation.nonstream.works") - def test_missing_query_returns_error( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: - tool = _register_search_tool(proxy, resources) - key = resources.key() - result = proxy.transport.send( - "/v1/search", - headers=proxy.transport.bearer(key), - json=SearchRequest(search_tool_name=tool, max_results=3), - ) - assert_client_error(result, "search missing query") - - @pytest.mark.covers("llm.search.openai.input_validation.nonstream.works") - def test_empty_query_returns_client_error( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: - tool = _register_search_tool(proxy, resources) - key = resources.key() - result = proxy.transport.send( - "/v1/search", - headers=proxy.transport.bearer(key), - json=SearchRequest(search_tool_name=tool, query=""), - ) - assert_client_error(result, "search empty query") - - @pytest.mark.covers("llm.search.openai.input_validation.nonstream.works") - def test_invalid_max_results_returns_client_error( - self, proxy: ProxyClient, resources: ResourceManager - ) -> None: - tool = _register_search_tool(proxy, resources) - key = resources.key() - result = proxy.transport.send( - "/v1/search", - headers=proxy.transport.bearer(key), - json=SearchRequest(search_tool_name=tool, query="tech trends", max_results=-1), - ) - assert_client_error(result, "search invalid max_results")