mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
test(timeouts): type deployment fixtures and isolate exception assertions
This commit is contained in:
parent
ea8cb33d64
commit
e3ed158750
2 changed files with 56 additions and 49 deletions
|
|
@ -8,13 +8,14 @@ import openai
|
|||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.types.router import DeploymentTypedDict
|
||||
from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
|
||||
|
||||
_ESSAY_MESSAGES: Final = [{"role": "user", "content": "hello, write a 20 pg essay"}]
|
||||
_TIMEOUT_SECONDS: Final = 0.5
|
||||
|
||||
|
||||
def _slow_openai_deployment(model_name: str) -> dict:
|
||||
def _slow_openai_deployment(model_name: str) -> DeploymentTypedDict:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
|
|
@ -25,7 +26,7 @@ def _slow_openai_deployment(model_name: str) -> dict:
|
|||
}
|
||||
|
||||
|
||||
def _slow_azure_deployment(model_name: str) -> dict:
|
||||
def _slow_azure_deployment(model_name: str) -> DeploymentTypedDict:
|
||||
return {
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
|
|
@ -124,16 +125,16 @@ def test_hanging_request_openai():
|
|||
def test_timeout_streaming():
|
||||
litellm.set_verbose = False
|
||||
with pytest.raises(openai.APITimeoutError):
|
||||
response = litellm.completion(
|
||||
model="openai/slow-endpoint",
|
||||
messages=_ESSAY_MESSAGES,
|
||||
api_base=FAKE_OPENAI_API_BASE,
|
||||
api_key="fake-key",
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
stream=True,
|
||||
tuple(
|
||||
litellm.completion(
|
||||
model="openai/slow-endpoint",
|
||||
messages=_ESSAY_MESSAGES,
|
||||
api_base=FAKE_OPENAI_API_BASE,
|
||||
api_key="fake-key",
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
stream=True,
|
||||
)
|
||||
)
|
||||
for chunk in response:
|
||||
print(chunk)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="local test")
|
||||
|
|
@ -162,6 +163,12 @@ def test_timeout_ollama():
|
|||
# test_timeout_ollama()
|
||||
|
||||
|
||||
async def _consume_async_response(response: litellm.ModelResponse | litellm.CustomStreamWrapper) -> None:
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
async for _ in response:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
@pytest.mark.parametrize("sync_mode", [True, False])
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -175,14 +182,9 @@ async def test_anthropic_timeout(streaming, sync_mode):
|
|||
"timeout": _TIMEOUT_SECONDS,
|
||||
"stream": streaming,
|
||||
}
|
||||
with pytest.raises(openai.APITimeoutError):
|
||||
if sync_mode:
|
||||
response = litellm.completion(**request)
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
for _ in response:
|
||||
pass
|
||||
else:
|
||||
response = await litellm.acompletion(**request)
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
async for _ in response:
|
||||
pass
|
||||
if sync_mode:
|
||||
with pytest.raises(openai.APITimeoutError):
|
||||
tuple(litellm.completion(**request))
|
||||
else:
|
||||
with pytest.raises(openai.APITimeoutError):
|
||||
await _consume_async_response(await litellm.acompletion(**request))
|
||||
|
|
|
|||
|
|
@ -3336,8 +3336,8 @@ _SYNC_TRANSPORT_TIMEOUT_CASES: Final = (
|
|||
def test_completion_timeout_reaches_the_transport_and_maps_to_timeout(case: _TransportTimeoutCase):
|
||||
seen_timeouts: list[object] = []
|
||||
client: Final = case.make_client(_timing_out_transport(seen_timeouts))
|
||||
with pytest.raises(litellm.Timeout):
|
||||
if case.embedding:
|
||||
if case.embedding:
|
||||
with pytest.raises(litellm.Timeout):
|
||||
litellm.embedding(
|
||||
model=case.model,
|
||||
input="hello",
|
||||
|
|
@ -3347,20 +3347,20 @@ def test_completion_timeout_reaches_the_transport_and_maps_to_timeout(case: _Tra
|
|||
max_retries=0,
|
||||
**case.extra_kwargs,
|
||||
)
|
||||
else:
|
||||
response = litellm.completion(
|
||||
model=case.model,
|
||||
messages=_TRANSPORT_TIMEOUT_MESSAGES,
|
||||
api_key="sk-test",
|
||||
client=client,
|
||||
timeout=_TRANSPORT_TIMEOUT_SECONDS,
|
||||
max_retries=0,
|
||||
stream=case.stream,
|
||||
**case.extra_kwargs,
|
||||
else:
|
||||
with pytest.raises(litellm.Timeout):
|
||||
tuple(
|
||||
litellm.completion(
|
||||
model=case.model,
|
||||
messages=_TRANSPORT_TIMEOUT_MESSAGES,
|
||||
api_key="sk-test",
|
||||
client=client,
|
||||
timeout=_TRANSPORT_TIMEOUT_SECONDS,
|
||||
max_retries=0,
|
||||
stream=case.stream,
|
||||
**case.extra_kwargs,
|
||||
)
|
||||
)
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
for _ in response:
|
||||
pass
|
||||
_assert_deadline_reached_transport(seen_timeouts)
|
||||
|
||||
|
||||
|
|
@ -3400,23 +3400,28 @@ _ASYNC_TRANSPORT_TIMEOUT_CASES: Final = (
|
|||
)
|
||||
|
||||
|
||||
async def _consume_async_timeout_response(response: litellm.ModelResponse | litellm.CustomStreamWrapper) -> None:
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
async for _ in response:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("case", _ASYNC_TRANSPORT_TIMEOUT_CASES, ids=lambda case: case.name)
|
||||
async def test_acompletion_timeout_reaches_the_transport_and_maps_to_timeout(case: _AsyncTransportTimeoutCase):
|
||||
seen_timeouts: list[object] = []
|
||||
client: Final = await case.make_client(_timing_out_transport(seen_timeouts))
|
||||
with pytest.raises(litellm.Timeout):
|
||||
response = await litellm.acompletion(
|
||||
model=case.model,
|
||||
messages=_TRANSPORT_TIMEOUT_MESSAGES,
|
||||
api_key="sk-test",
|
||||
client=client,
|
||||
timeout=_TRANSPORT_TIMEOUT_SECONDS,
|
||||
max_retries=0,
|
||||
stream=case.stream,
|
||||
**case.extra_kwargs,
|
||||
await _consume_async_timeout_response(
|
||||
await litellm.acompletion(
|
||||
model=case.model,
|
||||
messages=_TRANSPORT_TIMEOUT_MESSAGES,
|
||||
api_key="sk-test",
|
||||
client=client,
|
||||
timeout=_TRANSPORT_TIMEOUT_SECONDS,
|
||||
max_retries=0,
|
||||
stream=case.stream,
|
||||
**case.extra_kwargs,
|
||||
)
|
||||
)
|
||||
if isinstance(response, litellm.CustomStreamWrapper):
|
||||
async for _ in response:
|
||||
pass
|
||||
_assert_deadline_reached_transport(seen_timeouts)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue