mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
test(e2e): drop search e2e suite from vendor strategy PR
Remove the /v1/search coverage file and its registry rows so this PR no longer carries search endpoint testing.
This commit is contained in:
mubashir1osmani
parent
2e9d83d3b9
commit
8bbcd98cfa
3 changed files with 0 additions and 185 deletions
|
|
@ -39,8 +39,6 @@
|
|||
- {id: llm.realtime.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: realtime, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.19 / LIT-4778", rationale: "HTTP /v1/realtime/client_secrets and /calls reachable with auth"}
|
||||
- {id: llm.vector_stores.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store create/list/retrieve/delete lifecycle"}
|
||||
- {id: llm.vector_stores.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: vector_stores, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.17 / LIT-4778", rationale: "Vector store search and invalid id errors"}
|
||||
- {id: llm.search.openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: search, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.14 / LIT-4778", rationale: "POST /v1/search returns results for a registered tool"}
|
||||
- {id: llm.search.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: search, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.14 / LIT-4778", rationale: "Search missing/empty/invalid query rejected"}
|
||||
- {id: llm.bedrock_native.bedrock_converse.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse happy path"}
|
||||
- {id: llm.bedrock_native.bedrock_converse.basic.stream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: basic, streaming: stream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock native converse-stream"}
|
||||
- {id: llm.bedrock_native.bedrock_converse.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: bedrock_native, route: bedrock_converse, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.12 / LIT-4778", rationale: "Bedrock converse missing/empty messages and invalid model"}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ LlmEndpoint = Literal[
|
|||
"moderations",
|
||||
"realtime",
|
||||
"vector_stores",
|
||||
"search",
|
||||
"ocr",
|
||||
"bedrock_native",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,182 +0,0 @@
|
|||
"""Vendor §9.14: POST /v1/search through a registered search tool (LIT-4778).
|
||||
|
||||
Registers a Perplexity-backed search tool at runtime, runs a basic search, and
|
||||
pins missing/empty/invalid query handling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import NoBody, unwrap, assert_client_error
|
||||
from lifecycle import ResourceManager
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
|
||||
class SearchToolLiteLLMParams(BaseModel):
|
||||
search_provider: str
|
||||
|
||||
|
||||
class SearchToolBody(BaseModel):
|
||||
search_tool_name: str
|
||||
litellm_params: SearchToolLiteLLMParams
|
||||
search_tool_info: dict[str, str] | None = None
|
||||
|
||||
|
||||
class CreateSearchToolRequest(BaseModel):
|
||||
search_tool: SearchToolBody
|
||||
|
||||
|
||||
class SearchToolResponse(BaseModel):
|
||||
search_tool_id: str | None = None
|
||||
search_tool_name: str | None = None
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
search_tool_name: str | None = None
|
||||
query: str | None = None
|
||||
max_results: int | None = None
|
||||
country: str | None = None
|
||||
|
||||
|
||||
class SearchResultItem(BaseModel):
|
||||
title: str | None = None
|
||||
url: str | None = None
|
||||
|
||||
|
||||
class SearchResponse(BaseModel):
|
||||
object: str | None = None
|
||||
results: list[SearchResultItem] = []
|
||||
|
||||
|
||||
def _search_provider() -> str:
|
||||
if os.environ.get("PERPLEXITY_API_KEY") or os.environ.get("PERPLEXITYAI_API_KEY"):
|
||||
return "perplexity"
|
||||
if os.environ.get("TAVILY_API_KEY"):
|
||||
return "tavily"
|
||||
pytest.fail("set PERPLEXITY_API_KEY or TAVILY_API_KEY for /v1/search e2e coverage")
|
||||
|
||||
|
||||
def _register_search_tool(proxy: ProxyClient, resources: ResourceManager) -> str:
|
||||
provider = _search_provider()
|
||||
name = f"e2e-search-{unique_marker()}"
|
||||
created = unwrap(
|
||||
proxy.transport.post(
|
||||
"/search_tools",
|
||||
headers=proxy.transport.master,
|
||||
json=CreateSearchToolRequest(
|
||||
search_tool=SearchToolBody(
|
||||
search_tool_name=name,
|
||||
litellm_params=SearchToolLiteLLMParams(search_provider=provider),
|
||||
search_tool_info={"description": "e2e search tool"},
|
||||
)
|
||||
),
|
||||
response_type=SearchToolResponse,
|
||||
)
|
||||
)
|
||||
tool_id = created.search_tool_id
|
||||
if tool_id is not None:
|
||||
def _delete_tool() -> None:
|
||||
_ = proxy.transport.delete(
|
||||
f"/search_tools/{tool_id}",
|
||||
headers=proxy.transport.master,
|
||||
json=NoBody(),
|
||||
response_type=NoBody,
|
||||
)
|
||||
|
||||
resources.defer(_delete_tool)
|
||||
return name
|
||||
|
||||
|
||||
class TestSearch:
|
||||
@pytest.mark.covers("llm.search.openai.basic.nonstream.works")
|
||||
def test_basic_search_returns_results(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
tool = _register_search_tool(proxy, resources)
|
||||
key = resources.key()
|
||||
result = unwrap(
|
||||
proxy.transport.post(
|
||||
"/v1/search",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SearchRequest(
|
||||
search_tool_name=tool,
|
||||
query="latest AI news",
|
||||
max_results=3,
|
||||
country="US",
|
||||
),
|
||||
response_type=SearchResponse,
|
||||
)
|
||||
)
|
||||
assert result.object in (None, "search")
|
||||
assert result.results, f"search returned no results: {result}"
|
||||
assert any(item.title or item.url for item in result.results), (
|
||||
f"search results lack title/url: {result}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.search.openai.basic.nonstream.works")
|
||||
@pytest.mark.parametrize("max_results", [1, 5, 10])
|
||||
def test_max_results_boundaries(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, max_results: int
|
||||
) -> None:
|
||||
tool = _register_search_tool(proxy, resources)
|
||||
key = resources.key()
|
||||
result = unwrap(
|
||||
proxy.transport.post(
|
||||
"/v1/search",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SearchRequest(
|
||||
search_tool_name=tool, query="weather forecast", max_results=max_results
|
||||
),
|
||||
response_type=SearchResponse,
|
||||
)
|
||||
)
|
||||
assert result.results, f"search returned no results for max_results={max_results}"
|
||||
assert len(result.results) <= max_results, (
|
||||
f"expected <= {max_results} results, got {len(result.results)}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.search.openai.input_validation.nonstream.works")
|
||||
def test_missing_query_returns_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
tool = _register_search_tool(proxy, resources)
|
||||
key = resources.key()
|
||||
result = proxy.transport.send(
|
||||
"/v1/search",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SearchRequest(search_tool_name=tool, max_results=3),
|
||||
)
|
||||
assert_client_error(result, "search missing query")
|
||||
|
||||
@pytest.mark.covers("llm.search.openai.input_validation.nonstream.works")
|
||||
def test_empty_query_returns_client_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
tool = _register_search_tool(proxy, resources)
|
||||
key = resources.key()
|
||||
result = proxy.transport.send(
|
||||
"/v1/search",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SearchRequest(search_tool_name=tool, query=""),
|
||||
)
|
||||
assert_client_error(result, "search empty query")
|
||||
|
||||
@pytest.mark.covers("llm.search.openai.input_validation.nonstream.works")
|
||||
def test_invalid_max_results_returns_client_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
tool = _register_search_tool(proxy, resources)
|
||||
key = resources.key()
|
||||
result = proxy.transport.send(
|
||||
"/v1/search",
|
||||
headers=proxy.transport.bearer(key),
|
||||
json=SearchRequest(search_tool_name=tool, query="tech trends", max_results=-1),
|
||||
)
|
||||
assert_client_error(result, "search invalid max_results")
|
||||
Loading…
Add table
Reference in a new issue