diff --git a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py index 87bd32d8dab..363b2a7e02e 100644 --- a/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py +++ b/tests/e2e/llm_translation/test_chat_completions_regression_e2e.py @@ -45,6 +45,10 @@ pytestmark = pytest.mark.e2e COHERE_BACKEND = "cohere/command-r-08-2024" GEMINI_BACKEND = "gemini/gemini-2.5-flash" +VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" +AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" +AZURE_OPENAI_API_VERSION: Final = "v1" +AZURE_FOUNDRY_BACKEND: Final = "azure_ai/claude-haiku-4-5" OPENAI_BACKEND = "openai/gpt-5.6" ANTHROPIC_BACKEND = "anthropic/claude-haiku-4-5-20251001" BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" @@ -108,7 +112,7 @@ def _assert_describes_cat(response: ChatResponse) -> None: assert response.choices, f"vision returned no choices: {response}" message = response.choices[0].message content = (message.content if message else None) or "" - assert "cat" in content.lower() or "feline" in content.lower(), ( + assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), ( f"vision response did not describe the image: {content[:200]}" ) @@ -208,7 +212,6 @@ class TestChatCompletionsRegression: @pytest.mark.covers( "llm.chat_completions.openai.basic.nonstream.works", "llm.chat_completions.anthropic.basic.nonstream.works", - "llm.chat_completions.vertex.basic.nonstream.works", exercised_on=[], ) def test_chat_returns_real_completion( @@ -336,6 +339,232 @@ class TestGeminiChatCompletions: assert row.status == "success", f"gemini chat spend status={row.status!r}" +class TestVertexChatCompletions: + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=VERTEX_BACKEND, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.vertex.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"vertex chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"vertex chat returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.vertex.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="What is the weather in San Francisco? Use the get_weather tool.", + ) + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + @pytest.mark.covers( + "llm.chat_completions.vertex.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + + @pytest.mark.covers( + "llm.chat_completions.vertex.basic.stream.works", + exercised_on=["chat_completions"], + ) + def test_vertex_chat_streams_real_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-vertex-stream") + key = resources.key() + + result = client.proxy.chat_stream( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Count from 1 to 5, one number per line. {unique_marker()}", + ) + ], + max_tokens=64, + stream=True, + ), + ) + _assert_streamed_completion(result) + + +class TestAzureOpenAIChatCompletions: + def _register(self, client: PassthroughClient, resources: ResourceManager, prefix: str) -> str: + model = f"{prefix}-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=AZURE_OPENAI_BACKEND, + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version=AZURE_OPENAI_API_VERSION, + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + return model + + @pytest.mark.covers( + "llm.chat_completions.azure_openai.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_openai_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-azure-openai-chat") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"azure openai chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"azure openai chat returned empty content: {response}" + + @pytest.mark.covers( + "llm.chat_completions.azure_openai.tool_use.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_openai_chat_returns_tool_call( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-azure-openai-tool") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="What is the weather in San Francisco? Use the get_weather tool.", + ) + ], + tools=[_WEATHER_TOOL], + tool_choice="required", + max_tokens=128, + ), + ) + ) + _assert_weather_tool_call(response) + + +class TestAzureFoundryChatCompletions: + @pytest.mark.covers( + "llm.chat_completions.azure_foundry.basic.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_azure_foundry_chat_returns_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = f"e2e-azure-foundry-chat-{unique_marker()}" + model_id = client.proxy.create_model( + model, + LiteLLMParamsBody( + model=AZURE_FOUNDRY_BACKEND, + api_base="os.environ/AZURE_AI_API_BASE", + api_key="os.environ/AZURE_AI_API_KEY", + ), + ) + resources.defer(lambda: client.proxy.delete_model(model_id)) + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=f"Reply with the single word pong. {unique_marker()}", + ) + ], + max_tokens=32, + ), + ) + ) + assert response.choices, f"azure foundry chat returned no choices: {response}" + content = response.choices[0].message.content if response.choices[0].message else None + assert content and content.strip(), f"azure foundry chat returned empty content: {response}" + + class TestHostedVllmChat: """hosted_vllm (self-hosted OpenAI-compatible server) via /chat/completions.""" @@ -764,6 +993,90 @@ class TestAnthropicChatCompletions: resources.defer(lambda: client.proxy.delete_model(model_id)) return model + @pytest.mark.covers( + "llm.chat_completions.anthropic.structured_output.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_structured_output_conforms_to_schema( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-schema") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content="Extract the person. John Doe is 42 years old.", + ) + ], + response_format=_PERSON_SCHEMA, + max_tokens=128, + ), + ) + ) + assert response.choices, f"anthropic structured output returned no choices: {response}" + message = response.choices[0].message + content = message.content if message else None + assert content, f"anthropic structured output returned empty content: {response}" + person = _Person.model_validate_json(content) + assert person.name.strip() and person.age == 42, f"anthropic schema output was wrong: {person}" + + @pytest.mark.covers( + "llm.chat_completions.anthropic.thinking.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_returns_thinking_content( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-thinking") + key = resources.key() + + response = unwrap( + client.proxy.chat( + key, + ChatBody( + model=model, + messages=[ + ChatMessage( + role="user", + content=( + "Prove that the sum of two odd integers is even, then find the smallest prime " + "greater than 100 such that p+2 is also prime." + ), + ) + ], + thinking=ThinkingParam(type="enabled", budget_tokens=1024), + max_tokens=2048, + ), + ) + ) + assert response.choices, f"anthropic thinking returned no choices: {response}" + message = response.choices[0].message + assert message and message.content and message.content.strip(), ( + f"anthropic thinking returned no answer content: {response}" + ) + assert message.reasoning_content and message.reasoning_content.strip(), ( + f"anthropic thinking returned no reasoning content: {response}" + ) + + @pytest.mark.covers( + "llm.chat_completions.anthropic.vision.nonstream.works", + exercised_on=["chat_completions"], + ) + def test_anthropic_chat_vision_describes_image( + self, client: PassthroughClient, resources: ResourceManager + ) -> None: + model = self._register(client, resources, "e2e-anthropic-vision") + key = resources.key() + + response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32))) + _assert_describes_cat(response) + @pytest.mark.covers( "llm.chat_completions.anthropic.basic.stream.works", exercised_on=["chat_completions"], diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 9cc70da63b0..6fa77694eb8 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -45,6 +45,9 @@ class _OptionalResponsesBody(BaseModel): BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" +AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" +AZURE_OPENAI_API_VERSION: Final = "v1" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" BEDROCK_EDGE_REGION: Final = "us-east-1" @@ -105,6 +108,23 @@ def _bedrock_params() -> LiteLLMParamsBody: ) +def _vertex_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=VERTEX_BACKEND, + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + ) + + +def _azure_openai_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=AZURE_OPENAI_BACKEND, + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version=AZURE_OPENAI_API_VERSION, + ) + + def _register( proxy: ProxyClient, resources: ResourceManager, params: LiteLLMParamsBody, prefix: str = "e2e-responses" ) -> str: @@ -291,6 +311,66 @@ class TestResponses: ) _assert_weather_call(response) + @pytest.mark.covers("llm.responses.vertex.basic.nonstream.works") + def test_responses_vertex_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) + assert response.output_text.strip(), f"/responses over vertex returned no output text: {response.output!r}" + + @pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works") + def test_responses_vertex_returns_function_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _vertex_params(), prefix="e2e-responses-vertex-tool") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + _assert_weather_call(response) + + @pytest.mark.covers("llm.responses.azure_openai.basic.nonstream.works") + def test_responses_azure_openai_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) + assert response.output_text.strip(), ( + f"/responses over azure openai returned no output text: {response.output!r}" + ) + + @pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works") + def test_responses_azure_openai_returns_function_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai-tool") + client = sdk.openai(resources.key()) + + response = client.responses.create( + model=model, + input="What is the weather in San Francisco? Use the get_weather tool.", + instructions=INSTRUCTIONS, + tools=[WEATHER_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + _assert_weather_call(response) + @pytest.mark.provider_edge_host @pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"]) def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field(