mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-13 08:52:02 +00:00
test(e2e): cover Vertex responses/embeddings and team model allowlist
Enterprise gateway profile gaps: /v1/responses and /embeddings on Vertex Gemini, plus team-scoped key model allowlist enforcement on chat
This commit is contained in:
parent
4333c37f11
commit
d1a98ffe6d
3 changed files with 110 additions and 4 deletions
|
|
@ -1,9 +1,7 @@
|
|||
"""Live e2e: POST /embeddings returns a real vector.
|
||||
|
||||
Registers an OpenAI embedding deployment at runtime and asserts a non-empty,
|
||||
non-zero vector came back. Migrated from
|
||||
litellm-regression-tests/tests/test_inference_endpoints.py; the LIT-3167 guard in
|
||||
tests/e2e/embeddings/ covers the Gemini embedding path.
|
||||
Registers embedding deployments at runtime (OpenAI + Vertex) and asserts a
|
||||
non-empty, non-zero vector came back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -20,6 +18,7 @@ pytestmark = pytest.mark.e2e
|
|||
|
||||
|
||||
class TestEmbeddingsEndpoint:
|
||||
@pytest.mark.covers("llm.embeddings.openai.basic.nonstream.works")
|
||||
def test_embeddings_returns_vector(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
|
|
@ -40,3 +39,28 @@ class TestEmbeddingsEndpoint:
|
|||
assert any(component != 0.0 for component in parsed.first_vector), (
|
||||
f"embedding vector is all zeros: {result.body[:300]}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.embeddings.vertex.basic.nonstream.works")
|
||||
def test_vertex_embeddings_returns_vector(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-embeddings-vertex-{unique_marker()}"
|
||||
model_id = endpoints_client.create_model(
|
||||
model,
|
||||
LiteLLMParamsBody(
|
||||
model="vertex_ai/text-embedding-005",
|
||||
vertex_project="os.environ/VERTEXAI_PROJECT",
|
||||
vertex_location="us-central1",
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
),
|
||||
)
|
||||
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
result = endpoints_client.embeddings(key, model, "Say this is a test!")
|
||||
require_successful_call(result)
|
||||
parsed = EmbeddingsResult.model_validate_json(result.body)
|
||||
assert parsed.first_vector, f"vertex /embeddings returned no vector: {result.body[:300]}"
|
||||
assert any(component != 0.0 for component in parsed.first_vector), (
|
||||
f"vertex embedding vector is all zeros: {result.body[:300]}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -237,6 +237,74 @@ class TestResponses:
|
|||
f"got {result.response_cost!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.responses.vertex.basic.nonstream.works")
|
||||
def test_responses_vertex_returns_completion(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-responses-vertex-{unique_marker()}"
|
||||
model_id = endpoints_client.create_model(model, _vertex_gemini_params())
|
||||
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
result = endpoints_client.responses(key, model, "reply with one word")
|
||||
require_successful_call(result)
|
||||
assert result.call_id, "Responses→Vertex must set x-litellm-call-id"
|
||||
parsed = ResponsesResult.model_validate_json(result.body)
|
||||
assert parsed.text.strip(), f"/responses vertex returned no output text: {result.body[:300]}"
|
||||
assert result.response_cost is not None and result.response_cost > 0, (
|
||||
f"Responses→Vertex must set x-litellm-response-cost > 0; got {result.response_cost!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("llm.responses.vertex.tool_use.nonstream.works")
|
||||
def test_responses_vertex_returns_function_call(
|
||||
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
||||
) -> None:
|
||||
model = f"e2e-responses-vertex-{unique_marker()}"
|
||||
model_id = endpoints_client.create_model(model, _vertex_gemini_params())
|
||||
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
result = endpoints_client.responses_with_tools(
|
||||
key,
|
||||
model,
|
||||
"What is the weather in San Francisco? Use the get_weather tool.",
|
||||
[
|
||||
ResponsesFunctionTool(
|
||||
name="get_weather",
|
||||
description="Get the weather for a location",
|
||||
parameters=FunctionParameters(
|
||||
properties={"location": FunctionParameterProperty(type="string")},
|
||||
required=["location"],
|
||||
),
|
||||
)
|
||||
],
|
||||
)
|
||||
require_successful_call(result)
|
||||
parsed = ResponsesResult.model_validate_json(result.body)
|
||||
function_call = next(
|
||||
(call for call in parsed.function_calls if call.name == "get_weather"),
|
||||
None,
|
||||
)
|
||||
assert function_call is not None, (
|
||||
f"vertex responses returned no get_weather function call: {result.body[:500]}"
|
||||
)
|
||||
assert function_call.arguments is not None
|
||||
raw_arguments = cast(object, json.loads(function_call.arguments))
|
||||
arguments = WeatherArguments.model_validate(raw_arguments)
|
||||
assert arguments.location, (
|
||||
f"vertex function call arguments missing location: {function_call.arguments}"
|
||||
)
|
||||
|
||||
|
||||
def _vertex_gemini_params() -> LiteLLMParamsBody:
|
||||
"""Vertex Gemini via gateway-held SA (os.environ/* resolved on the proxy)."""
|
||||
return LiteLLMParamsBody(
|
||||
model="vertex_ai/gemini-2.5-flash",
|
||||
vertex_project="os.environ/VERTEXAI_PROJECT",
|
||||
vertex_location="us-central1",
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
)
|
||||
|
||||
|
||||
def _parse_stream_event(
|
||||
event: str,
|
||||
|
|
|
|||
|
|
@ -189,6 +189,20 @@ class TestTeamRoutes:
|
|||
f"key generated under team {team_id} carries team_id {key_info.team_id!r} in /key/info"
|
||||
)
|
||||
|
||||
@pytest.mark.covers("mgmt.team.new.persists")
|
||||
def test_team_model_allowlist_enforced_on_chat(
|
||||
self, client: ManagementClient, resources: ResourceManager
|
||||
) -> None:
|
||||
"""Team-scoped keys only reach models on the team's allow-list (central for
|
||||
org/team access control). Allowed model is callable; a sibling model is 403."""
|
||||
team_id = _create_team(
|
||||
client, resources, f"e2e-mgmt-team-{unique_marker()}", ["gemini-2.5-flash"]
|
||||
)
|
||||
key = _generate_key(client, resources, KeyGenerateBody(team_id=team_id))
|
||||
|
||||
_poll_chat_ok(client, key, "gemini-2.5-flash")
|
||||
_poll_chat_denied(client, key, "gpt-5.5")
|
||||
|
||||
@pytest.mark.covers("mgmt.team.member_add.persists")
|
||||
def test_member_add_and_delete_persist_to_team_info(
|
||||
self, client: ManagementClient, resources: ResourceManager
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue