mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-09 08:18:12 +00:00
c943d650f4
* test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag router, batches and mcp tests with Subject metadata and record client steps * test(e2e): leave the batches cleanup harness unit tests untagged * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): name every driven model on the vllm batch, prompt caching and complexity router subjects
56 lines
2.7 KiB
Python
56 lines
2.7 KiB
Python
"""Live e2e: a per-request timeout surfaces to the caller instead of hanging.
|
|
|
|
A deployment created with a 1ms deadline always exceeds it against the real
|
|
backend. With no fallback in play, the proxy must return the timeout to the
|
|
caller: a 408 for a non-streamed request, and the same timeout surfaced on the
|
|
streamed path (either a 408 before the stream opens or a timeout error carried in
|
|
the response).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from complexity_router_client import ComplexityRouterClient
|
|
from e2e_config import unique_marker
|
|
from e2e_metadata import Domain, Mode, Provider, Subject, meta
|
|
from lifecycle import ResourceManager
|
|
from reliability_support import REAL_MODEL, chat_override, create_timeout_deployment
|
|
|
|
pytestmark = pytest.mark.e2e
|
|
|
|
|
|
class TestReliabilityTimeouts:
|
|
@pytest.mark.covers("reliability.timeout.request_timeout.exceeds_deadline")
|
|
@meta(Subject(domain=Domain.ROUTING, providers=(Provider.OPENAI,), models=(REAL_MODEL,), mode=Mode.NONSTREAM))
|
|
def test_request_timeout_exceeds_deadline(
|
|
self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
|
|
) -> None:
|
|
name = f"reliability-timeout-{unique_marker()}"
|
|
model_id = create_timeout_deployment(client.proxy, name)
|
|
resources.defer(lambda: client.proxy.delete_model(model_id))
|
|
|
|
resp = chat_override(client.proxy, scoped_key, name, "hello")
|
|
assert resp.status_code == 408, (
|
|
f"a timed-out request should return 408, got {resp.status_code}: {resp.body[:300]}"
|
|
)
|
|
assert "timeout" in resp.body.lower(), f"the 408 body should name the timeout, got: {resp.body[:300]}"
|
|
|
|
@pytest.mark.covers("reliability.timeout.stream_timeout.exceeds_deadline")
|
|
@meta(Subject(domain=Domain.ROUTING, providers=(Provider.OPENAI,), models=(REAL_MODEL,), mode=Mode.STREAM))
|
|
def test_stream_timeout_exceeds_deadline(
|
|
self, client: ComplexityRouterClient, resources: ResourceManager, scoped_key: str
|
|
) -> None:
|
|
name = f"reliability-stream-timeout-{unique_marker()}"
|
|
model_id = create_timeout_deployment(client.proxy, name)
|
|
resources.defer(lambda: client.proxy.delete_model(model_id))
|
|
|
|
resp = chat_override(client.proxy, scoped_key, name, "hello", stream=True)
|
|
surfaced = f"{resp.body} {resp.stream_error or ''}".lower()
|
|
assert resp.status_code >= 400, (
|
|
f"a timed-out streaming request should surface an error status, got {resp.status_code}: {resp.body[:300]}"
|
|
)
|
|
assert "timeout" in surfaced, (
|
|
f"the streamed timeout error should name the timeout, got body={resp.body[:300]}, "
|
|
f"stream_error={resp.stream_error!r}"
|
|
)
|