mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
a780d4e4e3
* test(e2e): cover customer chat/messages cost + streaming paths Fills five uncovered P0 registry cells matching the customer's confirmed stack (OpenAI SDK, Bedrock, /v1/messages) and their per-request cost dependency: - /v1/messages logs cost that matches the x-litellm-response-cost header (LIT-4076) - OpenAI /chat/completions streams real content, and a non-streamed call is costed - Bedrock Converse /chat/completions returns real content non-streamed and streamed The streaming checks aggregate delta content and parse every chunk as JSON, so a clean-but-empty stream or a truncated chunk fails instead of passing on a bare 200. * test(e2e): add tool-use coverage for openai, bedrock converse, anthropic responses Function-calling regression guards on the paths the customer's agentic SDK usage exercises: OpenAI and Bedrock Converse /chat/completions, and Anthropic /v1/responses. The model is forced to call a weather tool and the test asserts the returned tool call names the function and carries JSON-parseable arguments with the expected field, so a dropped tool_call or malformed argument JSON fails instead of passing on a bare 200. Adds a minimal tool_calls field to the response OutMessage. * test(e2e): cover bedrock converse responses + thinking Adds llm.responses.bedrock_converse.basic/tool_use and llm.chat_completions.bedrock_converse.thinking. The thinking test enables extended thinking and requires reasoning_content plus a real answer, so a path that drops the reasoning block fails rather than passing. * test(e2e): cover bedrock embeddings + openai structured output and reasoning Bedrock Titan embeddings return a real vector; OpenAI structured output must yield schema-conforming JSON with the correct extracted values (age==42, not just valid JSON); an OpenAI reasoning call must report reasoning tokens, so a non-reasoning fallback fails. Adds response_format to ChatBody and reasoning-token details to Usage. * test(e2e): cover vision + streaming tool calls on openai and bedrock converse Vision on both providers must describe the image (not just 200); the streamed OpenAI tool call is reassembled from its fragments and its argument JSON parsed, so a stream that never completes the call or splits its JSON fails. Extends ChatMessage content to a typed text/image union. * test(e2e): cover openai prompt caching hit on repeated large prefix A repeated large-prefix prompt must report cached prompt tokens on the second call, so a cache regression that stops reusing the prefix (and silently re-bills full input) fails here. * test(e2e): cover openai audio speech + bedrock rerank and image generation Marks the OpenAI TTS cell and adds Bedrock Titan rerank (top_n honored, scored) and Bedrock Titan image generation (returns b64/url), the customer's non-chat AWS surfaces. * test(e2e): cover end-user (customer) create persistence mgmt.end_user.new.happy_path: create an end-user via /customer/new and confirm /customer/info reports it, the end-user-identity surface the customer relies on for per-customer controls. Adds customer models + management-client methods. * test(e2e): enforce key model allow-list on the passthrough route other.auth.passthrough.model_allowlist_enforced: a key scoped to gemini must be denied a claude call through the anthropic passthrough route (403), so custom-auth scoping is not bypassable by going through passthrough instead of /chat/completions. * test(e2e): address Greptile - assert stream data events, correlate messages spend by key - streaming: assert len(stream_events) > 1 instead of chunks > 1, since chunks counts the terminal data: [DONE] marker and would pass a single content event - messages cost: correlate the spend row by the unique scoped key rather than the Anthropic response id, which need not equal the proxy spend-log request_id
75 lines
3 KiB
Python
75 lines
3 KiB
Python
"""Live e2e: POST /v1/rerank ranks documents by relevance.
|
|
|
|
Registers a Cohere rerank deployment at runtime and asserts the endpoint returns
|
|
scored results within the requested top_n. Migrated from
|
|
litellm-regression-tests/tests/test_inference_endpoints.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from e2e_config import require_env, unique_marker
|
|
from e2e_http import require_successful_call
|
|
from endpoints_client import EndpointsClient, RerankResult
|
|
from lifecycle import ResourceManager
|
|
from models import LiteLLMParamsBody
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
DOCUMENTS = [
|
|
"Carson City is the capital city of the American state of Nevada.",
|
|
"The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean.",
|
|
"Washington, D.C. is the capital of the United States.",
|
|
"Capital punishment has existed in the United States since before it was a country.",
|
|
]
|
|
QUERY = "What is the capital of the United States?"
|
|
|
|
|
|
def _assert_top_n_scored(body: str) -> None:
|
|
parsed = RerankResult.model_validate_json(body)
|
|
assert parsed.results, f"/rerank returned no results: {body[:300]}"
|
|
assert len(parsed.results) <= 3, f"top_n=3 not honored: {body[:300]}"
|
|
assert parsed.results[0].relevance_score is not None, (
|
|
f"top rerank result has no relevance_score: {body[:300]}"
|
|
)
|
|
|
|
|
|
class TestRerank:
|
|
@pytest.mark.covers("llm.rerank.cohere.basic.nonstream.works")
|
|
def test_rerank_scores_top_n(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-rerank-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="cohere/rerank-v3.5", api_key="os.environ/COHERE_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
|
|
require_successful_call(result)
|
|
_assert_top_n_scored(result.body)
|
|
|
|
@pytest.mark.covers("llm.rerank.bedrock.basic.nonstream.works", exercised_on=["rerank"])
|
|
def test_bedrock_rerank_scores_top_n(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
require_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_REGION")
|
|
model = f"e2e-bedrock-rerank-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(
|
|
model="bedrock/amazon.rerank-v1:0",
|
|
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
|
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
|
aws_region_name="os.environ/AWS_REGION",
|
|
),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.rerank(key, model, QUERY, DOCUMENTS, top_n=3)
|
|
require_successful_call(result)
|
|
_assert_top_n_scored(result.body)
|