diff --git a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py index 56f2de8bd4f..d365beee84b 100644 --- a/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py +++ b/tests/e2e/llm_translation/test_embeddings_endpoint_e2e.py @@ -1,9 +1,7 @@ """Live e2e: POST /embeddings returns a real vector. -Registers an OpenAI embedding deployment at runtime and asserts a non-empty, -non-zero vector came back. Migrated from -litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in -tests/e2e/embeddings/ covers the Gemini embedding path. +Registers embedding deployments at runtime (OpenAI + Vertex) and asserts a +non-empty, non-zero vector came back. """ from __future__ import annotations @@ -20,6 +18,7 @@ pytestmark = pytest.mark.e2e class TestEmbeddingsEndpoint: + @pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works") def test_embeddings_returns_vector( self, endpoints_client: EndpointsClient, resources: ResourceManager ) -> None: @@ -40,3 +39,28 @@ class TestEmbeddingsEndpoint: assert any(component != 0.0 for component in parsed.first_vector), ( f"embedding vector is all zeros: {result.body[:300]}" ) + + @pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works") + def test_vertex_embeddings_returns_vector( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-embeddings-vertex-{unique_marker()}" + model_id = endpoints_client.create_model( + model, + LiteLLMParamsBody( + model="vertex_ai/text-embedding-005", + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ), + ) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.embeddings(key, model, "Say this is a test!") + require_successful_call(result) + parsed = EmbeddingsResult.model_validate_json(result.body) + assert parsed.first_vector, f"vertex /embeddings returned no vector: {result.body[:300]}" + assert any(component != 0.0 for component in parsed.first_vector), ( + f"vertex embedding vector is all zeros: {result.body[:300]}" + ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 2188bcc7f29..350748ca3d3 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -237,6 +237,74 @@ class TestResponses: f"got {result.response_cost!r}" ) + @pytest.mark.covers("llm.responses.vertex.basic.nonstream.works") + def test_responses_vertex_returns_completion( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-vertex-{unique_marker()}" + model_id = endpoints_client.create_model(model, _vertex_gemini_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses(key, model, "reply with one word") + require_successful_call(result) + assert result.call_id, "Responses→Vertex must set x-litellm-call-id" + parsed = ResponsesResult.model_validate_json(result.body) + assert parsed.text.strip(), f"/responses vertex returned no output text: {result.body[:300]}" + assert result.response_cost is not None and result.response_cost > 0, ( + f"Responses→Vertex must set x-litellm-response-cost > 0; got {result.response_cost!r}" + ) + + @pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works") + def test_responses_vertex_returns_function_call( + self, endpoints_client: EndpointsClient, resources: ResourceManager + ) -> None: + model = f"e2e-responses-vertex-{unique_marker()}" + model_id = endpoints_client.create_model(model, _vertex_gemini_params()) + resources.defer(lambda: endpoints_client.delete_model(model_id)) + key = resources.key() + + result = endpoints_client.responses_with_tools( + key, + model, + "What is the weather in San Francisco? Use the get_weather tool.", + [ + ResponsesFunctionTool( + name="get_weather", + description="Get the weather for a location", + parameters=FunctionParameters( + properties={"location": FunctionParameterProperty(type="string")}, + required=["location"], + ), + ) + ], + ) + require_successful_call(result) + parsed = ResponsesResult.model_validate_json(result.body) + function_call = next( + (call for call in parsed.function_calls if call.name == "get_weather"), + None, + ) + assert function_call is not None, ( + f"vertex responses returned no get_weather function call: {result.body[:500]}" + ) + assert function_call.arguments is not None + raw_arguments = cast(object, json.loads(function_call.arguments)) + arguments = WeatherArguments.model_validate(raw_arguments) + assert arguments.location, ( + f"vertex function call arguments missing location: {function_call.arguments}" + ) + + +def _vertex_gemini_params() -> LiteLLMParamsBody: + """Vertex Gemini via gateway-held SA (os.environ/* resolved on the proxy).""" + return LiteLLMParamsBody( + model="vertex_ai/gemini-2.5-flash", + vertex_project="os.environ/VERTEXAI_PROJECT", + vertex_location="us-central1", + vertex_credentials="os.environ/VERTEXAI_CREDENTIALS", + ) + def _parse_stream_event( event: str, diff --git a/tests/e2e/management/test_management_e2e.py b/tests/e2e/management/test_management_e2e.py index adbf3e8b065..34a499ab364 100644 --- a/tests/e2e/management/test_management_e2e.py +++ b/tests/e2e/management/test_management_e2e.py @@ -189,6 +189,20 @@ class TestTeamRoutes: f"key generated under team {team_id} carries team_id {key_info.team_id!r} in /key/info" ) + @pytest.mark.covers("mgmt.team.new.persists") + def test_team_model_allowlist_enforced_on_chat( + self, client: ManagementClient, resources: ResourceManager + ) -> None: + """Team-scoped keys only reach models on the team's allow-list (central for + org/team access control). Allowed model is callable; a sibling model is 403.""" + team_id = _create_team( + client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"] + ) + key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id)) + + _poll_chat_ok(client, key, "gemini-2.5-flash") + _poll_chat_denied(client, key, "gpt-5.5") + @pytest.mark.covers("mgmt.team.member_add.persists") def test_member_add_and_delete_persist_to_team_info( self, client: ManagementClient, resources: ResourceManager