scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-25 01:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 80

test(e2e): cover chat and responses registry gaps

Browse source
This commit is contained in:
Yuneng Jiang 2026-09-18 02:43:22 -07:00
parent 8fc9c46d1a
commit 3af44daf6d
No known key found for this signature in database
3 changed files with 447 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
tests/e2e/llm_translation/endpoints_client.py
View file

@ -66,6 +66,7 @@ class ResponsesInputMessage(BaseModel):
ResponsesInput = str | list[ResponsesInputMessage]
ResponsesToolChoice = Literal["auto", "required", "none"]
class ResponsesRequest(BaseModel):
@ -74,6 +75,7 @@ class ResponsesRequest(BaseModel):
instructions: str | None = None
stream: bool = False
tools: list[ResponsesFunctionTool] | None = None
tool_choice: ResponsesToolChoice | None = None
guardrails: list[str] | None = None
cache: dict[str, bool] | None = {"no-cache": True}
@ -351,7 +353,13 @@ class EndpointsClient:
)
def responses_with_tools(
self, key: str, model: str, text: str, tools: list[ResponsesFunctionTool]
self,
key: str,
model: str,
text: str,
tools: list[ResponsesFunctionTool],
*,
tool_choice: ResponsesToolChoice | None = None,
) -> StreamingResponse:
return self._send(
"/v1/responses",
@ -361,6 +369,7 @@ class EndpointsClient:
input=text,
instructions="You are a helpful assistant",
tools=tools,
tool_choice=tool_choice,
),
)

313
tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
View file

@ -45,6 +45,9 @@ pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.6-sol"
AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5"
OPENAI_BACKEND = "openai/gpt-5.6"
ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001"
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
@ -208,7 +211,6 @@ class TestChatCompletionsRegression:
@pytest.mark.covers(
"llm.chat_completions.openai.basic.nonstream.works",
"llm.chat_completions.anthropic.basic.nonstream.works",
"llm.chat_completions.vertex.basic.nonstream.works",
exercised_on=[],
)
def test_chat_returns_real_completion(
@ -336,6 +338,231 @@ class TestGeminiChatCompletions:
assert row.status == "success", f"gemini chat spend status={row.status!r}"
class TestVertexChatCompletions:
def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.vertex.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-chat")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"vertex chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"vertex chat returned empty content: {response}"
@pytest.mark.covers(
"llm.chat_completions.vertex.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_returns_tool_call(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-tool")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="What is the weather in San Francisco? Use the get_weather tool.",
)
],
tools=[_WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
),
)
)
_assert_weather_tool_call(response)
@pytest.mark.covers(
"llm.chat_completions.vertex.vision.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_vision_describes_image(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
_assert_describes_cat(response)
@pytest.mark.covers(
"llm.chat_completions.vertex.basic.stream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_streams_real_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-stream")
key = resources.key()
result = client.proxy.chat_stream(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Count from 1 to 5, one number per line. {unique_marker()}",
)
],
max_tokens=64,
stream=True,
),
)
_assert_streamed_completion(result)
class TestAzureOpenAIChatCompletions:
def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.azure_openai.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_openai_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-azure-openai-chat")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"azure openai chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"azure openai chat returned empty content: {response}"
@pytest.mark.covers(
"llm.chat_completions.azure_openai.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_openai_chat_returns_tool_call(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-azure-openai-tool")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="What is the weather in San Francisco? Use the get_weather tool.",
)
],
tools=[_WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
),
)
)
_assert_weather_tool_call(response)
class TestAzureFoundryChatCompletions:
@pytest.mark.covers(
"llm.chat_completions.azure_foundry.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_foundry_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = f"e2e-azure-foundry-chat-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=AZURE_FOUNDRY_BACKEND,
api_base="os.environ/AZURE_AI_API_BASE",
api_key="os.environ/AZURE_AI_API_KEY",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"azure foundry chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"azure foundry chat returned empty content: {response}"
class TestHostedVllmChat:
"""hosted_vllm (self-hosted OpenAI-compatible server) via /chat/completions."""
@ -764,6 +991,90 @@ class TestAnthropicChatCompletions:
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.anthropic.structured_output.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_structured_output_conforms_to_schema(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-schema")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="Extract the person. John Doe is 42 years old.",
)
],
response_format=_PERSON_SCHEMA,
max_tokens=128,
),
)
)
assert response.choices, f"anthropic structured output returned no choices: {response}"
message = response.choices[0].message
content = message.content if message else None
assert content, f"anthropic structured output returned empty content: {response}"
person = _Person.model_validate_json(content)
assert person.name.strip() and person.age == 42, f"anthropic schema output was wrong: {person}"
@pytest.mark.covers(
"llm.chat_completions.anthropic.thinking.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_returns_thinking_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-thinking")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=(
"Prove that the sum of two odd integers is even, then find the smallest prime "
"greater than 100 such that p+2 is also prime."
),
)
],
thinking=ThinkingParam(type="enabled", budget_tokens=1024),
max_tokens=2048,
),
)
)
assert response.choices, f"anthropic thinking returned no choices: {response}"
message = response.choices[0].message
assert message and message.content and message.content.strip(), (
f"anthropic thinking returned no answer content: {response}"
)
assert message.reasoning_content and message.reasoning_content.strip(), (
f"anthropic thinking returned no reasoning content: {response}"
)
@pytest.mark.covers(
"llm.chat_completions.anthropic.vision.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_vision_describes_image(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
_assert_describes_cat(response)
@pytest.mark.covers(
"llm.chat_completions.anthropic.basic.stream.works",
exercised_on=["chat_completions"],

126
tests/e2e/llm_translation/test_responses_e2e.py
View file

@ -8,7 +8,7 @@ litellm-regression-tests/tests/test_inference_endpoints.py.
from __future__ import annotations
import json
from typing import cast
from typing import Final, cast
import pytest
from e2e_config import unique_marker
@ -39,6 +39,8 @@ class _OptionalResponsesBody(BaseModel):
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.6-sol"
WEATHER_TOOL = ResponsesFunctionTool(
name="get_weather",
@ -295,6 +297,128 @@ class TestResponses:
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
def _register(
self,
endpoints_client: EndpointsClient,
resources: ResourceManager,
prefix: str,
params: LiteLLMParamsBody,
) -> tuple[str, str]:
model = f"{prefix}-{unique_marker()}"
model_id = endpoints_client.create_model(model, params)
resources.defer(lambda: endpoints_client.delete_model(model_id))
return model, resources.key()
@pytest.mark.covers("llm.responses.vertex.basic.nonstream.works")
def test_responses_vertex_returns_completion(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model, key = self._register(
endpoints_client,
resources,
"e2e-responses-vertex",
LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
),
)
result = endpoints_client.responses(key, model, "reply with one word")
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
assert parsed.text.strip(), f"/responses over vertex returned no output text: {result.body[:300]}"
@pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works")
def test_responses_vertex_returns_function_call(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model, key = self._register(
endpoints_client,
resources,
"e2e-responses-vertex-tool",
LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
),
)
result = endpoints_client.responses_with_tools(
key,
model,
"What is the weather in San Francisco? Use the get_weather tool.",
[WEATHER_TOOL],
tool_choice="required",
)
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
function_call = next(
(call for call in parsed.function_calls if call.name == "get_weather"),
None,
)
assert function_call is not None, f"no vertex get_weather function call: {result.body[:500]}"
assert function_call.arguments is not None
raw_arguments = cast(object, json.loads(function_call.arguments))
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"vertex function call arguments missing location: {function_call.arguments}"
@pytest.mark.covers("llm.responses.azure_openai.basic.nonstream.works")
def test_responses_azure_openai_returns_completion(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model, key = self._register(
endpoints_client,
resources,
"e2e-responses-azure-openai",
LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
),
)
result = endpoints_client.responses(key, model, "reply with one word")
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
assert parsed.text.strip(), (
f"/responses over azure openai returned no output text: {result.body[:300]}"
)
@pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works")
def test_responses_azure_openai_returns_function_call(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model, key = self._register(
endpoints_client,
resources,
"e2e-responses-azure-openai-tool",
LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
),
)
result = endpoints_client.responses_with_tools(
key,
model,
"What is the weather in San Francisco? Use the get_weather tool.",
[WEATHER_TOOL],
tool_choice="required",
)
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
function_call = next(
(call for call in parsed.function_calls if call.name == "get_weather"),
None,
)
assert function_call is not None, f"no azure openai get_weather function call: {result.body[:500]}"
assert function_call.arguments is not None
raw_arguments = cast(object, json.loads(function_call.arguments))
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, f"azure openai function call arguments missing location: {function_call.arguments}"
@pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400")
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
def test_missing_input_returns_error(