scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-24 00:52:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 91

Merge pull request #41794 from BerriAI/litellm_wt_0918_138e

Browse source 
test(e2e): cover chat and responses registry gaps
This commit is contained in:
yuneng-jiang 2026-09-21 17:27:09 -07:00 • committed by GitHub
commit 79c7136292
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 395 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

317
tests/e2e/llm_translation/test_chat_completions_regression_e2e.py
View file

@ -45,6 +45,10 @@ pytestmark = pytest.mark.e2e
COHERE_BACKEND = "cohere/command-r-08-2024"
GEMINI_BACKEND = "gemini/gemini-2.5-flash"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5"
OPENAI_BACKEND = "openai/gpt-5.6"
ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001"
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
@ -108,7 +112,7 @@ def _assert_describes_cat(response: ChatResponse) -> None:
assert response.choices, f"vision returned no choices: {response}"
message = response.choices[0].message
content = (message.content if message else None) or ""
assert "cat" in content.lower() or "feline" in content.lower(), (
assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), (
f"vision response did not describe the image: {content[:200]}"
)
@ -208,7 +212,6 @@ class TestChatCompletionsRegression:
@pytest.mark.covers(
"llm.chat_completions.openai.basic.nonstream.works",
"llm.chat_completions.anthropic.basic.nonstream.works",
"llm.chat_completions.vertex.basic.nonstream.works",
exercised_on=[],
)
def test_chat_returns_real_completion(
@ -336,6 +339,232 @@ class TestGeminiChatCompletions:
assert row.status == "success", f"gemini chat spend status={row.status!r}"
class TestVertexChatCompletions:
def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.vertex.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-chat")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"vertex chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"vertex chat returned empty content: {response}"
@pytest.mark.covers(
"llm.chat_completions.vertex.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_returns_tool_call(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-tool")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="What is the weather in San Francisco? Use the get_weather tool.",
)
],
tools=[_WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
),
)
)
_assert_weather_tool_call(response)
@pytest.mark.covers(
"llm.chat_completions.vertex.vision.nonstream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_vision_describes_image(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
_assert_describes_cat(response)
@pytest.mark.covers(
"llm.chat_completions.vertex.basic.stream.works",
exercised_on=["chat_completions"],
)
def test_vertex_chat_streams_real_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-vertex-stream")
key = resources.key()
result = client.proxy.chat_stream(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Count from 1 to 5, one number per line. {unique_marker()}",
)
],
max_tokens=64,
stream=True,
),
)
_assert_streamed_completion(result)
class TestAzureOpenAIChatCompletions:
def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str:
model = f"{prefix}-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
api_version=AZURE_OPENAI_API_VERSION,
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.azure_openai.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_openai_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-azure-openai-chat")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"azure openai chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"azure openai chat returned empty content: {response}"
@pytest.mark.covers(
"llm.chat_completions.azure_openai.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_openai_chat_returns_tool_call(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-azure-openai-tool")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="What is the weather in San Francisco? Use the get_weather tool.",
)
],
tools=[_WEATHER_TOOL],
tool_choice="required",
max_tokens=128,
),
)
)
_assert_weather_tool_call(response)
class TestAzureFoundryChatCompletions:
@pytest.mark.covers(
"llm.chat_completions.azure_foundry.basic.nonstream.works",
exercised_on=["chat_completions"],
)
def test_azure_foundry_chat_returns_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = f"e2e-azure-foundry-chat-{unique_marker()}"
model_id = client.proxy.create_model(
model,
LiteLLMParamsBody(
model=AZURE_FOUNDRY_BACKEND,
api_base="os.environ/AZURE_AI_API_BASE",
api_key="os.environ/AZURE_AI_API_KEY",
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=f"Reply with the single word pong. {unique_marker()}",
)
],
max_tokens=32,
),
)
)
assert response.choices, f"azure foundry chat returned no choices: {response}"
content = response.choices[0].message.content if response.choices[0].message else None
assert content and content.strip(), f"azure foundry chat returned empty content: {response}"
class TestHostedVllmChat:
"""hosted_vllm (self-hosted OpenAI-compatible server) via /chat/completions."""
@ -764,6 +993,90 @@ class TestAnthropicChatCompletions:
resources.defer(lambda: client.proxy.delete_model(model_id))
return model
@pytest.mark.covers(
"llm.chat_completions.anthropic.structured_output.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_structured_output_conforms_to_schema(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-schema")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content="Extract the person. John Doe is 42 years old.",
)
],
response_format=_PERSON_SCHEMA,
max_tokens=128,
),
)
)
assert response.choices, f"anthropic structured output returned no choices: {response}"
message = response.choices[0].message
content = message.content if message else None
assert content, f"anthropic structured output returned empty content: {response}"
person = _Person.model_validate_json(content)
assert person.name.strip() and person.age == 42, f"anthropic schema output was wrong: {person}"
@pytest.mark.covers(
"llm.chat_completions.anthropic.thinking.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_returns_thinking_content(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-thinking")
key = resources.key()
response = unwrap(
client.proxy.chat(
key,
ChatBody(
model=model,
messages=[
ChatMessage(
role="user",
content=(
"Prove that the sum of two odd integers is even, then find the smallest prime "
"greater than 100 such that p+2 is also prime."
),
)
],
thinking=ThinkingParam(type="enabled", budget_tokens=1024),
max_tokens=2048,
),
)
)
assert response.choices, f"anthropic thinking returned no choices: {response}"
message = response.choices[0].message
assert message and message.content and message.content.strip(), (
f"anthropic thinking returned no answer content: {response}"
)
assert message.reasoning_content and message.reasoning_content.strip(), (
f"anthropic thinking returned no reasoning content: {response}"
)
@pytest.mark.covers(
"llm.chat_completions.anthropic.vision.nonstream.works",
exercised_on=["chat_completions"],
)
def test_anthropic_chat_vision_describes_image(
self, client: PassthroughClient, resources: ResourceManager
) -> None:
model = self._register(client, resources, "e2e-anthropic-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
_assert_describes_cat(response)
@pytest.mark.covers(
"llm.chat_completions.anthropic.basic.stream.works",
exercised_on=["chat_completions"],

80
tests/e2e/llm_translation/test_responses_e2e.py
View file

@ -45,6 +45,9 @@ class _OptionalResponsesBody(BaseModel):
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash"
AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano"
AZURE_OPENAI_API_VERSION: Final = "v1"
INSTRUCTIONS = "You are a helpful assistant"
CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg"
BEDROCK_EDGE_REGION: Final = "us-east-1"
@ -105,6 +108,23 @@ def _bedrock_params() -> LiteLLMParamsBody:
)
def _vertex_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=VERTEX_BACKEND,
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
)
def _azure_openai_params() -> LiteLLMParamsBody:
return LiteLLMParamsBody(
model=AZURE_OPENAI_BACKEND,
api_base="os.environ/AZURE_API_BASE",
api_key="os.environ/AZURE_API_KEY",
api_version=AZURE_OPENAI_API_VERSION,
)
def _register(
proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody, prefix: str = "e2e-responses"
) -> str:
@ -291,6 +311,66 @@ class TestResponses:
)
_assert_weather_call(response)
@pytest.mark.covers("llm.responses.vertex.basic.nonstream.works")
def test_responses_vertex_returns_completion(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex")
client = sdk.openai(resources.key())
response = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
assert response.output_text.strip(), f"/responses over vertex returned no output text: {response.output!r}"
@pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works")
def test_responses_vertex_returns_function_call(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex-tool")
client = sdk.openai(resources.key())
response = client.responses.create(
model=model,
input="What is the weather in San Francisco? Use the get_weather tool.",
instructions=INSTRUCTIONS,
tools=[WEATHER_TOOL],
tool_choice="required",
extra_body=NO_PROXY_CACHE,
)
_assert_weather_call(response)
@pytest.mark.covers("llm.responses.azure_openai.basic.nonstream.works")
def test_responses_azure_openai_returns_completion(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai")
client = sdk.openai(resources.key())
response = client.responses.create(
model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE
)
assert response.output_text.strip(), (
f"/responses over azure openai returned no output text: {response.output!r}"
)
@pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works")
def test_responses_azure_openai_returns_function_call(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai-tool")
client = sdk.openai(resources.key())
response = client.responses.create(
model=model,
input="What is the weather in San Francisco? Use the get_weather tool.",
instructions=INSTRUCTIONS,
tools=[WEATHER_TOOL],
tool_choice="required",
extra_body=NO_PROXY_CACHE,
)
_assert_weather_call(response)
@pytest.mark.provider_edge_host
@pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"])
def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field(