test(e2e): cover Vertex responses/embeddings and team model allowlist

Enterprise gateway profile gaps: /v1/responses and /embeddings on Vertex
Gemini, plus team-scoped key model allowlist enforcement on chat
This commit is contained in:
mubashir1osmani 2026-07-18 17:43:51 -07:00
parent 4333c37f11
commit d1a98ffe6d
3 changed files with 110 additions and 4 deletions

View file

@ -1,9 +1,7 @@
"""Live e2e: POST /embeddings returns a real vector.
Registers an OpenAI embedding deployment at runtime and asserts a non-empty,
non-zero vector came back. Migrated from
litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in
tests/e2e/embeddings/ covers the Gemini embedding path.
Registers embedding deployments at runtime (OpenAI + Vertex) and asserts a
non-empty, non-zero vector came back.
"""
from __future__ import annotations
@ -20,6 +18,7 @@ pytestmark = pytest.mark.e2e
class TestEmbeddingsEndpoint:
@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works")
def test_embeddings_returns_vector(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
@ -40,3 +39,28 @@ class TestEmbeddingsEndpoint:
assert any(component != 0.0 for component in parsed.first_vector), (
f"embedding vector is all zeros: {result.body[:300]}"
)
@pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works")
def test_vertex_embeddings_returns_vector(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-embeddings-vertex-{unique_marker()}"
model_id = endpoints_client.create_model(
model,
LiteLLMParamsBody(
model="vertex_ai/text-embedding-005",
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
),
)
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
result = endpoints_client.embeddings(key, model, "Say this is a test!")
require_successful_call(result)
parsed = EmbeddingsResult.model_validate_json(result.body)
assert parsed.first_vector, f"vertex /embeddings returned no vector: {result.body[:300]}"
assert any(component != 0.0 for component in parsed.first_vector), (
f"vertex embedding vector is all zeros: {result.body[:300]}"
)

View file

@ -237,6 +237,74 @@ class TestResponses:
f"got {result.response_cost!r}"
)
@pytest.mark.covers("llm.responses.vertex.basic.nonstream.works")
def test_responses_vertex_returns_completion(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-responses-vertex-{unique_marker()}"
model_id = endpoints_client.create_model(model, _vertex_gemini_params())
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
result = endpoints_client.responses(key, model, "reply with one word")
require_successful_call(result)
assert result.call_id, "Responses→Vertex must set x-litellm-call-id"
parsed = ResponsesResult.model_validate_json(result.body)
assert parsed.text.strip(), f"/responses vertex returned no output text: {result.body[:300]}"
assert result.response_cost is not None and result.response_cost > 0, (
f"Responses→Vertex must set x-litellm-response-cost > 0; got {result.response_cost!r}"
)
@pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works")
def test_responses_vertex_returns_function_call(
self, endpoints_client: EndpointsClient, resources: ResourceManager
) -> None:
model = f"e2e-responses-vertex-{unique_marker()}"
model_id = endpoints_client.create_model(model, _vertex_gemini_params())
resources.defer(lambda: endpoints_client.delete_model(model_id))
key = resources.key()
result = endpoints_client.responses_with_tools(
key,
model,
"What is the weather in San Francisco? Use the get_weather tool.",
[
ResponsesFunctionTool(
name="get_weather",
description="Get the weather for a location",
parameters=FunctionParameters(
properties={"location": FunctionParameterProperty(type="string")},
required=["location"],
),
)
],
)
require_successful_call(result)
parsed = ResponsesResult.model_validate_json(result.body)
function_call = next(
(call for call in parsed.function_calls if call.name == "get_weather"),
None,
)
assert function_call is not None, (
f"vertex responses returned no get_weather function call: {result.body[:500]}"
)
assert function_call.arguments is not None
raw_arguments = cast(object, json.loads(function_call.arguments))
arguments = WeatherArguments.model_validate(raw_arguments)
assert arguments.location, (
f"vertex function call arguments missing location: {function_call.arguments}"
)
def _vertex_gemini_params() -> LiteLLMParamsBody:
"""Vertex Gemini via gateway-held SA (os.environ/* resolved on the proxy)."""
return LiteLLMParamsBody(
model="vertex_ai/gemini-2.5-flash",
vertex_project="os.environ/VERTEXAI_PROJECT",
vertex_location="us-central1",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
)
def _parse_stream_event(
event: str,

View file

@ -189,6 +189,20 @@ class TestTeamRoutes:
f"key generated under team {team_id} carries team_id {key_info.team_id!r} in /key/info"
)
@pytest.mark.covers("mgmt.team.new.persists")
def test_team_model_allowlist_enforced_on_chat(
self, client: ManagementClient, resources: ResourceManager
) -> None:
"""Team-scoped keys only reach models on the team's allow-list (central for
org/team access control). Allowed model is callable; a sibling model is 403."""
team_id = _create_team(
client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"]
)
key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id))
_poll_chat_ok(client, key, "gemini-2.5-flash")
_poll_chat_denied(client, key, "gpt-5.5")
@pytest.mark.covers("mgmt.team.member_add.persists")
def test_member_add_and_delete_persist_to_team_info(
self, client: ManagementClient, resources: ResourceManager