From c70e4857fae0a9e9b0deb2aeeaf534e1136cef3a Mon Sep 17 00:00:00 2001 From: Yuneng Jiang Date: Thu, 3 Sep 2026 09:53:30 -0700 Subject: [PATCH] test(timeout): time out against the local fake endpoint instead of api.openai.com test_router_timeout, test_timeout_streaming and test_openai_embedding_timeouts asked api.openai.com for a response in 10 to 100 microseconds and asserted the resulting exception was a timeout. No connect can finish in that window, so socket.create_connection always walked the whole address list, and because it re-raises only the LAST address's error, the assertion was decided by the order getaddrinfo happened to return. api.openai.com is dual-stack and the CI container has no usable IPv6, so a trailing AAAA record made the last attempt fail with an OSError. httpcore maps socket.timeout to ConnectTimeout but OSError to ConnectError, so the expected APITimeoutError arrived as APIConnectionError and the job went red. The three tests were really measuring DNS ordering, not litellm. Point them at the fake OpenAI endpoint the suite already runs, ask for the slow-endpoint model it already delays on, and give them a timeout comfortably under that delay. The embeddings route did not honour slow-endpoint yet, so it now delays the same way chat and text completions already do. Each test also gained a failure on the success path. Without it a request that returned instead of timing out fell out of the try block and the test passed on a result it was written to reject. --- tests/_fake_openai_endpoint_server.py | 5 ++++- tests/local_testing/test_embedding.py | 8 ++++++-- tests/local_testing/test_router.py | 7 ++++--- tests/local_testing/test_timeout.py | 8 ++++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/_fake_openai_endpoint_server.py b/tests/_fake_openai_endpoint_server.py index ac83e74b66a..d3af31aeef5 100644 --- a/tests/_fake_openai_endpoint_server.py +++ b/tests/_fake_openai_endpoint_server.py @@ -207,13 +207,16 @@ async def completions(request: Request) -> Response: async def embeddings(request: Request) -> Response: body = await _parse_body(request) + model = _requested_model(body) + if model == _SLOW_MODEL: + await asyncio.sleep(_SLOW_RESPONSE_SECONDS) raw_input = body.get("input", "") count = len(raw_input) if isinstance(raw_input, list) else 1 return JSONResponse( { "object": "list", "data": [{"object": "embedding", "index": i, "embedding": [0.0] * 1536} for i in range(max(count, 1))], - "model": _requested_model(body), + "model": model, "usage": {"prompt_tokens": 5, "total_tokens": 5}, } ) diff --git a/tests/local_testing/test_embedding.py b/tests/local_testing/test_embedding.py index ee2ac14f498..c119334da6f 100644 --- a/tests/local_testing/test_embedding.py +++ b/tests/local_testing/test_embedding.py @@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm from litellm import completion, completion_cost, embedding +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE litellm.set_verbose = False @@ -269,11 +270,14 @@ def test_openai_azure_embedding_timeouts(): def test_openai_embedding_timeouts(): try: response = embedding( - model="text-embedding-ada-002", + model="openai/slow-endpoint", input=["good morning from litellm"], - timeout=0.00001, + api_base=FAKE_OPENAI_API_BASE, + api_key="fake-key", + timeout=0.5, ) print(response) + pytest.fail("Expected timeout error, the request returned instead") except openai.APITimeoutError: print("Good job got OpenAI timeout error!") pass diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index c714bb4f9a7..bd0a9bf8df4 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -1552,8 +1552,9 @@ def test_router_timeout(): { "model_name": "gpt-3.5-turbo", "litellm_params": { - "model": "gpt-3.5-turbo", - "api_key": "os.environ/OPENAI_API_KEY", + "model": "openai/slow-endpoint", + "api_base": FAKE_OPENAI_API_BASE, + "api_key": "fake-key", }, } ] @@ -1562,7 +1563,7 @@ def test_router_timeout(): start_time = time.time() try: res = router.completion( - model="gpt-3.5-turbo", messages=messages, timeout=0.0001 + model="gpt-3.5-turbo", messages=messages, timeout=0.5 ) print(res) pytest.fail("this should have timed out") diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index 66054a0930a..784e2c73cd7 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -12,6 +12,7 @@ import openai import pytest import litellm +from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE @pytest.mark.parametrize( @@ -216,13 +217,16 @@ def test_timeout_streaming(): litellm.set_verbose = False try: response = litellm.completion( - model="gpt-3.5-turbo", + model="openai/slow-endpoint", messages=[{"role": "user", "content": "hello, write a 20 pg essay"}], - timeout=0.0001, + api_base=FAKE_OPENAI_API_BASE, + api_key="fake-key", + timeout=0.5, stream=True, ) for chunk in response: print(chunk) + pytest.fail("Did not raise error `openai.APITimeoutError`. The stream completed instead") except openai.APITimeoutError as e: print( "Passed: Raised correct exception. Got openai.APITimeoutError\nGood Job", e