test(timeout): time out against the local fake endpoint instead of api.openai.com

test_router_timeout, test_timeout_streaming and test_openai_embedding_timeouts
asked api.openai.com for a response in 10 to 100 microseconds and asserted the
resulting exception was a timeout. No connect can finish in that window, so
socket.create_connection always walked the whole address list, and because it
re-raises only the LAST address's error, the assertion was decided by the order
getaddrinfo happened to return.

api.openai.com is dual-stack and the CI container has no usable IPv6, so a
trailing AAAA record made the last attempt fail with an OSError. httpcore maps
socket.timeout to ConnectTimeout but OSError to ConnectError, so the expected
APITimeoutError arrived as APIConnectionError and the job went red. The three
tests were really measuring DNS ordering, not litellm.

Point them at the fake OpenAI endpoint the suite already runs, ask for the
slow-endpoint model it already delays on, and give them a timeout comfortably
under that delay. The embeddings route did not honour slow-endpoint yet, so it
now delays the same way chat and text completions already do.

Each test also gained a failure on the success path. Without it a request that
returned instead of timing out fell out of the try block and the test passed on
a result it was written to reject.

tests/_fake_openai_endpoint_server.py

@@ -207,13 +207,16 @@ async def completions(request: Request) -> Response:
 
 async def embeddings(request: Request) -> Response:
     body = await _parse_body(request)
+    model = _requested_model(body)
+    if model == _SLOW_MODEL:
+        await asyncio.sleep(_SLOW_RESPONSE_SECONDS)
     raw_input = body.get("input", "")
     count = len(raw_input) if isinstance(raw_input, list) else 1
     return JSONResponse(
         {
             "object": "list",
             "data": [{"object": "embedding", "index": i, "embedding": [0.0] * 1536} for i in range(max(count, 1))],
-            "model": _requested_model(body),
+            "model": model,
             "usage": {"prompt_tokens": 5, "total_tokens": 5},
         }
     )

tests/local_testing/test_embedding.py

@@ -15,6 +15,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 import litellm
 from litellm import completion, completion_cost, embedding
+from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
 
 litellm.set_verbose = False
 
@@ -269,11 +270,14 @@ def test_openai_azure_embedding_timeouts():
 def test_openai_embedding_timeouts():
     try:
         response = embedding(
-            model="text-embedding-ada-002",
+            model="openai/slow-endpoint",
             input=["good morning from litellm"],
-            timeout=0.00001,
+            api_base=FAKE_OPENAI_API_BASE,
+            api_key="fake-key",
+            timeout=0.5,
         )
         print(response)
+        pytest.fail("Expected timeout error, the request returned instead")
     except openai.APITimeoutError:
         print("Good job got OpenAI timeout error!")
         pass

tests/local_testing/test_router.py

@@ -1552,8 +1552,9 @@ def test_router_timeout():
         {
             "model_name": "gpt-3.5-turbo",
             "litellm_params": {
-                "model": "gpt-3.5-turbo",
-                "api_key": "os.environ/OPENAI_API_KEY",
+                "model": "openai/slow-endpoint",
+                "api_base": FAKE_OPENAI_API_BASE,
+                "api_key": "fake-key",
             },
         }
     ]
@@ -1562,7 +1563,7 @@ def test_router_timeout():
     start_time = time.time()
     try:
         res = router.completion(
-            model="gpt-3.5-turbo", messages=messages, timeout=0.0001
+            model="gpt-3.5-turbo", messages=messages, timeout=0.5
         )
         print(res)
         pytest.fail("this should have timed out")

tests/local_testing/test_timeout.py

@@ -12,6 +12,7 @@ import openai
 import pytest
 
 import litellm
+from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE
 
 
 @pytest.mark.parametrize(
@@ -216,13 +217,16 @@ def test_timeout_streaming():
     litellm.set_verbose = False
     try:
         response = litellm.completion(
-            model="gpt-3.5-turbo",
+            model="openai/slow-endpoint",
             messages=[{"role": "user", "content": "hello, write a 20 pg essay"}],
-            timeout=0.0001,
+            api_base=FAKE_OPENAI_API_BASE,
+            api_key="fake-key",
+            timeout=0.5,
             stream=True,
         )
         for chunk in response:
             print(chunk)
+        pytest.fail("Did not raise error `openai.APITimeoutError`. The stream completed instead")
     except openai.APITimeoutError as e:
         print(
             "Passed: Raised correct exception. Got openai.APITimeoutError\nGood Job", e