mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
433 lines
18 KiB
Python
433 lines
18 KiB
Python
"""Live e2e: POST /v1/responses returns a real completion.
|
|
|
|
Registers an OpenAI deployment at runtime, drives the Responses API through the
|
|
gateway, and asserts output text came back. Migrated from
|
|
litellm-regression-tests/tests/test_inference_endpoints.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import threading
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass, field
|
|
from types import MappingProxyType
|
|
from typing import Final, cast
|
|
|
|
import pytest
|
|
from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
|
|
from e2e_http import (
|
|
assert_client_error,
|
|
require_successful_call,
|
|
)
|
|
from endpoints_client import (
|
|
EndpointsClient,
|
|
FunctionParameterProperty,
|
|
FunctionParameters,
|
|
ResponsesFunctionTool,
|
|
ResponsesOutputTextDeltaEvent,
|
|
ResponsesResult,
|
|
ResponsesStreamEventType,
|
|
)
|
|
from lifecycle import ResourceManager
|
|
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
|
from provider_edge import LiveEdge, start_provider_edge
|
|
from provider_edge_bedrock import bedrock_signer
|
|
from pydantic import BaseModel, ValidationError
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
|
|
class _OptionalResponsesBody(BaseModel):
|
|
model: str | None = None
|
|
input: str | None = None
|
|
max_output_tokens: int | None = None
|
|
|
|
|
|
BEDROCK_CONVERSE_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
|
|
BEDROCK_EDGE_REGION: Final = "us-east-1"
|
|
BEDROCK_EDGE_MOUNT: Final = f"bedrock/{BEDROCK_EDGE_REGION}"
|
|
|
|
|
|
class ConverseRequestBody(BaseModel):
|
|
additionalModelRequestFields: dict[str, str] | None = None
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class ConverseRequestCapture:
|
|
"""The Converse bodies the proxy actually sent upstream, as seen by a live
|
|
edge sitting between the proxy and Bedrock."""
|
|
|
|
_bodies: list[ConverseRequestBody] = field(default_factory=list)
|
|
_lock: threading.Lock = field(default_factory=threading.Lock)
|
|
|
|
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
|
|
if body is None or "/converse" not in url:
|
|
return
|
|
with self._lock:
|
|
self._bodies.append(ConverseRequestBody.model_validate_json(body))
|
|
|
|
@property
|
|
def bodies(self) -> tuple[ConverseRequestBody, ...]:
|
|
with self._lock:
|
|
return tuple(self._bodies)
|
|
|
|
|
|
WEATHER_TOOL = ResponsesFunctionTool(
|
|
name="get_weather",
|
|
description="Get the weather for a location",
|
|
parameters=FunctionParameters(
|
|
properties={"location": FunctionParameterProperty(type="string")},
|
|
required=["location"],
|
|
),
|
|
)
|
|
|
|
|
|
def _bedrock_params() -> LiteLLMParamsBody:
|
|
return LiteLLMParamsBody(
|
|
model=BEDROCK_CONVERSE_BACKEND,
|
|
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
|
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
|
aws_region_name="os.environ/AWS_REGION",
|
|
)
|
|
|
|
|
|
class WeatherArguments(BaseModel):
|
|
location: str
|
|
|
|
|
|
class TestResponses:
|
|
@pytest.mark.covers("llm.responses.openai.basic.nonstream.works")
|
|
def test_responses_returns_completion(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses(key, model, "reply with one word")
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}"
|
|
|
|
@pytest.mark.covers("llm.responses.openai.basic.stream.works")
|
|
def test_responses_streaming_returns_completion(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses(key, model, "reply with one word", stream=True)
|
|
require_successful_call(result)
|
|
delta_events = tuple(
|
|
parsed
|
|
for event in result.stream_events
|
|
if (parsed := _parse_stream_event(event)) is not None
|
|
)
|
|
|
|
assert any(event.delta for event in delta_events), "responses stream returned no text deltas"
|
|
assert result.stream_events, "responses stream returned no events"
|
|
assert (
|
|
ResponsesStreamEventType.model_validate_json(result.stream_events[-1]).type
|
|
== "response.completed"
|
|
), "responses stream did not terminate with response.completed"
|
|
|
|
@pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged")
|
|
def test_responses_logs_cost(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses(key, model, f"reply with one word {unique_marker()}")
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}"
|
|
assert result.call_id and parsed.id, f"missing response identifiers: {result.body[:300]}"
|
|
|
|
rows = endpoints_client.proxy.poll_logs_for_request_id(
|
|
parsed.id,
|
|
predicate=lambda logged_rows: any((row.spend or 0) > 0 for row in logged_rows),
|
|
)
|
|
row = next((logged_row for logged_row in rows if (logged_row.spend or 0) > 0), None)
|
|
assert row is not None, f"no costed spend row for response id {parsed.id}"
|
|
assert "gpt-4o-mini" in (row.model or ""), f"unexpected spend row model: {row.model}"
|
|
|
|
@pytest.mark.covers("llm.responses.openai.tool_use.nonstream.works")
|
|
def test_responses_returns_function_call(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses_with_tools(
|
|
key,
|
|
model,
|
|
"What is the weather in San Francisco? Use the get_weather tool.",
|
|
[
|
|
ResponsesFunctionTool(
|
|
name="get_weather",
|
|
description="Get the weather for a location",
|
|
parameters=FunctionParameters(
|
|
properties={"location": FunctionParameterProperty(type="string")},
|
|
required=["location"],
|
|
),
|
|
)
|
|
],
|
|
)
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
function_call = next(
|
|
(call for call in parsed.function_calls if call.name == "get_weather"),
|
|
None,
|
|
)
|
|
assert function_call is not None, f"no get_weather function call: {result.body[:500]}"
|
|
assert function_call.arguments is not None
|
|
raw_arguments = cast(object, json.loads(function_call.arguments))
|
|
arguments = WeatherArguments.model_validate(raw_arguments)
|
|
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
|
|
|
|
@pytest.mark.covers("llm.responses.openai.vision.nonstream.works")
|
|
def test_responses_vision_describes_image(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses_vision(
|
|
key,
|
|
model,
|
|
"What animal is shown in this image? Answer in one word",
|
|
"https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg",
|
|
)
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
text = parsed.text.strip().lower()
|
|
assert text, f"/responses vision returned no output text: {result.body[:300]}"
|
|
assert any(
|
|
keyword in text
|
|
for keyword in ("cat", "feline")
|
|
), f"vision response did not describe the image: {parsed.text[:300]}"
|
|
|
|
@pytest.mark.covers("llm.responses.anthropic.basic.nonstream.works")
|
|
def test_responses_anthropic_returns_completion(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(
|
|
model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
|
|
),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses(key, model, "reply with one word")
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
assert parsed.text.strip(), f"/responses returned no output text: {result.body[:300]}"
|
|
|
|
@pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works")
|
|
def test_responses_anthropic_returns_function_call(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(
|
|
model="anthropic/claude-haiku-4-5", api_key="os.environ/ANTHROPIC_API_KEY"
|
|
),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses_with_tools(
|
|
key,
|
|
model,
|
|
"What is the weather in San Francisco? Use the get_weather tool.",
|
|
[
|
|
ResponsesFunctionTool(
|
|
name="get_weather",
|
|
description="Get the weather for a location",
|
|
parameters=FunctionParameters(
|
|
properties={"location": FunctionParameterProperty(type="string")},
|
|
required=["location"],
|
|
),
|
|
)
|
|
],
|
|
)
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
function_call = next(
|
|
(call for call in parsed.function_calls if call.name == "get_weather"),
|
|
None,
|
|
)
|
|
assert function_call is not None, f"no get_weather function call: {result.body[:500]}"
|
|
assert function_call.arguments is not None
|
|
raw_arguments = cast(object, json.loads(function_call.arguments))
|
|
arguments = WeatherArguments.model_validate(raw_arguments)
|
|
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
|
|
|
|
@pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works")
|
|
def test_responses_bedrock_returns_completion(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(model, _bedrock_params())
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses(key, model, "reply with one word")
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
assert parsed.text.strip(), f"/responses over bedrock returned no output text: {result.body[:300]}"
|
|
|
|
@pytest.mark.covers("llm.responses.bedrock_converse.tool_use.nonstream.works")
|
|
def test_responses_bedrock_returns_function_call(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(model, _bedrock_params())
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
|
|
result = endpoints_client.responses_with_tools(
|
|
key, model, "What is the weather in San Francisco? Use the get_weather tool.", [WEATHER_TOOL]
|
|
)
|
|
require_successful_call(result)
|
|
parsed = ResponsesResult.model_validate_json(result.body)
|
|
function_call = next((call for call in parsed.function_calls if call.name == "get_weather"), None)
|
|
assert function_call is not None, f"no get_weather function call over bedrock: {result.body[:500]}"
|
|
assert function_call.arguments is not None
|
|
raw_arguments = cast(object, json.loads(function_call.arguments))
|
|
arguments = WeatherArguments.model_validate(raw_arguments)
|
|
assert arguments.location, f"function call arguments missing location: {function_call.arguments}"
|
|
|
|
@pytest.mark.provider_edge_host
|
|
@pytest.mark.parametrize("endpoint", ["/v1/responses", "/v1/chat/completions"])
|
|
def test_bedrock_forwards_allowed_safety_identifier_as_additional_model_request_field(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager, endpoint: str
|
|
) -> None:
|
|
capture: Final = ConverseRequestCapture()
|
|
edge: Final = start_provider_edge(
|
|
LiveEdge(observe_request=capture.observe, sign=bedrock_signer(BEDROCK_EDGE_REGION)),
|
|
mounts=MappingProxyType({BEDROCK_EDGE_MOUNT: f"https://bedrock-runtime.{BEDROCK_EDGE_REGION}.amazonaws.com"}),
|
|
bind_host=PROVIDER_EDGE_BIND_HOST,
|
|
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
|
|
)
|
|
resources.defer(edge.shutdown)
|
|
model: Final = f"e2e-responses-{unique_marker()}"
|
|
model_id: Final = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(
|
|
model=BEDROCK_CONVERSE_BACKEND,
|
|
api_base=edge.edge.api_base(BEDROCK_EDGE_MOUNT),
|
|
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
|
|
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
|
|
aws_region_name=BEDROCK_EDGE_REGION,
|
|
allowed_openai_params=["safety_identifier"],
|
|
),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key: Final = resources.key()
|
|
safety_identifier: Final = f"end-user-{unique_marker()}"
|
|
|
|
if endpoint == "/v1/responses":
|
|
endpoints_client.responses(key, model, "reply with one word", safety_identifier=safety_identifier)
|
|
else:
|
|
endpoints_client.proxy.chat(
|
|
key,
|
|
ChatBody(
|
|
model=model,
|
|
messages=[ChatMessage(role="user", content="reply with one word")],
|
|
safety_identifier=safety_identifier,
|
|
),
|
|
)
|
|
|
|
forwarded: Final = tuple(body.additionalModelRequestFields for body in capture.bodies)
|
|
assert forwarded, f"{endpoint} produced no Bedrock Converse request"
|
|
assert forwarded == ({"safety_identifier": safety_identifier},) * len(forwarded), (
|
|
f"{endpoint} did not forward safety_identifier to Bedrock Converse on every attempt: {capture.bodies}"
|
|
)
|
|
|
|
@pytest.mark.skip(reason="stage red: product gap, /v1/responses 500s (aresponses TypeError) on missing input instead of 400")
|
|
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
|
|
def test_missing_input_returns_error(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-val-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
result = endpoints_client.proxy.transport.send(
|
|
"/v1/responses",
|
|
headers=endpoints_client.proxy.transport.bearer(key),
|
|
json=_OptionalResponsesBody(model=model),
|
|
)
|
|
assert_client_error(result, "responses missing input")
|
|
|
|
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
|
|
def test_missing_model_returns_client_error(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
key = resources.key()
|
|
result = endpoints_client.proxy.transport.send(
|
|
"/v1/responses",
|
|
headers=endpoints_client.proxy.transport.bearer(key),
|
|
json=_OptionalResponsesBody(input="ping"),
|
|
)
|
|
assert_client_error(result, "responses missing model")
|
|
|
|
@pytest.mark.covers("llm.responses.openai.input_validation.nonstream.works")
|
|
def test_empty_input_returns_client_error(
|
|
self, endpoints_client: EndpointsClient, resources: ResourceManager
|
|
) -> None:
|
|
model = f"e2e-responses-val-{unique_marker()}"
|
|
model_id = endpoints_client.create_model(
|
|
model,
|
|
LiteLLMParamsBody(model="openai/gpt-4o-mini", api_key="os.environ/OPENAI_API_KEY"),
|
|
)
|
|
resources.defer(lambda: endpoints_client.delete_model(model_id))
|
|
key = resources.key()
|
|
result = endpoints_client.proxy.transport.send(
|
|
"/v1/responses",
|
|
headers=endpoints_client.proxy.transport.bearer(key),
|
|
json=_OptionalResponsesBody(model=model, input=""),
|
|
)
|
|
assert_client_error(result, "responses empty input")
|
|
|
|
def _parse_stream_event(
|
|
event: str,
|
|
) -> ResponsesOutputTextDeltaEvent | None:
|
|
try:
|
|
return ResponsesOutputTextDeltaEvent.model_validate_json(event)
|
|
except ValidationError:
|
|
return None
|