scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-07 08:26:10 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

Merge branch 'litellm_fix_timeout_test_fix' into litellm_merge_timeout_issue

Browse source
This commit is contained in:
Sameer Kankute 2026-01-28 08:56:29 +05:30 • committed by GitHub
commit 5276085f3c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 16 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
litellm/main.py
View file

@ -599,15 +599,8 @@ async def acompletion( # noqa: PLR0915
# Add the context to the function
ctx = contextvars.copy_context()
func_with_context = partial(ctx.run, func)
if timeout is not None and isinstance(timeout, (int, float)):
timeout_value = float(timeout)
init_response = await asyncio.wait_for(
loop.run_in_executor(None, func_with_context), timeout=timeout_value
)
else:
init_response = await loop.run_in_executor(None, func_with_context)
init_response = await loop.run_in_executor(None, func_with_context)
if isinstance(init_response, dict) or isinstance(
init_response, ModelResponse
): ## CACHING SCENARIO
@ -615,11 +608,7 @@ async def acompletion( # noqa: PLR0915
response = ModelResponse(**init_response)
response = init_response
elif asyncio.iscoroutine(init_response):
if timeout is not None and isinstance(timeout, (int, float)):
timeout_value = float(timeout)
response = await asyncio.wait_for(init_response, timeout=timeout_value)
else:
response = await init_response
response = await init_response
else:
response = init_response # type: ignore
@ -636,15 +625,6 @@ async def acompletion( # noqa: PLR0915
loop=loop
) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls)
return response
except asyncio.TimeoutError:
custom_llm_provider = custom_llm_provider or "openai"
from litellm.exceptions import Timeout
raise Timeout(
message=f"Request timed out after {timeout} seconds",
model=model,
llm_provider=custom_llm_provider,
)
except Exception as e:
custom_llm_provider = custom_llm_provider or "openai"
raise exception_type(

134
tests/local_testing/test_timeout.py
View file

@ -96,7 +96,6 @@ def test_bedrock_timeout():
def test_hanging_request_azure():
litellm.set_verbose = True
import asyncio
from unittest.mock import AsyncMock, patch, MagicMock
try:
router = litellm.Router(
@ -105,8 +104,8 @@ def test_hanging_request_azure():
"model_name": "azure-gpt",
"litellm_params": {
"model": "azure/gpt-4o-new-test",
"api_base": os.environ.get("AZURE_API_BASE", "https://test.openai.azure.com"),
"api_key": os.environ.get("AZURE_API_KEY", "test-key"),
"api_base": os.environ["AZURE_API_BASE"],
"api_key": os.environ["AZURE_API_KEY"],
},
},
{
@ -120,25 +119,16 @@ def test_hanging_request_azure():
encoded = litellm.utils.encode(model="gpt-3.5-turbo", text="blue")[0]
async def _test():
# Mock the Azure OpenAI client's create method to simulate a hanging request
with patch("openai.resources.chat.completions.AsyncCompletions.create") as mock_create:
# Simulate a hanging request that takes longer than the timeout
async def hanging_request(*args, **kwargs):
await asyncio.sleep(10) # Sleep much longer than the 0.01s timeout
return MagicMock()
mock_create.side_effect = hanging_request
response = await router.acompletion(
model="azure-gpt",
messages=[
{"role": "user", "content": f"what color is red {uuid.uuid4()}"}
],
logit_bias={encoded: 100},
timeout=0.01,
)
print(response)
return response
response = await router.acompletion(
model="azure-gpt",
messages=[
{"role": "user", "content": f"what color is red {uuid.uuid4()}"}
],
logit_bias={encoded: 100},
timeout=0.01,
)
print(response)
return response
response = asyncio.run(_test())
@ -150,16 +140,9 @@ def test_hanging_request_azure():
)
print(type(e))
pass
except litellm.exceptions.APIError as e:
# Azure may convert CancelledError to APIError - this is also acceptable for timeout scenarios
print(
"Passed: Raised APIError due to timeout (CancelledError). This is acceptable.", e
)
print(type(e))
pass
except Exception as e:
pytest.fail(
f"Did not raise error `openai.APITimeoutError` or `litellm.exceptions.APIError`. Instead raised error type: {type(e)}, Error: {e}"
f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}"
)
@ -302,94 +285,3 @@ async def test_anthropic_timeout(streaming, sync_mode):
)
print(type(e))
pass
@pytest.mark.asyncio
async def test_timeout_respects_total_time_not_per_retry():
"""
Test that timeout applies to the TOTAL operation time, not per-retry.
This test ensures that when a user sets timeout=2, the entire operation
(including all retries) times out at ~2 seconds, not at 2s * num_retries.
This is a regression test for the issue where timeout was being applied
per-retry attempt, causing the total time to be much longer than expected.
"""
litellm.set_verbose = False
timeout_value = 2.0
# Allow for some overhead (network, processing, etc.)
# but ensure we don't wait for multiple retries
max_allowed_time = timeout_value + 1.0 # 3 seconds max
start_time = time.time()
try:
# This should timeout because we're asking for a long response
# with a very short timeout
response = await litellm.acompletion(
model="gpt-3.5-turbo",
timeout=timeout_value,
messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}],
)
pytest.fail("Expected timeout error but got a response")
except (openai.APITimeoutError, litellm.exceptions.Timeout) as e:
elapsed_time = time.time() - start_time
print(f"Timeout occurred after {elapsed_time:.2f} seconds")
print(f"Expected timeout: {timeout_value} seconds")
print(f"Max allowed time: {max_allowed_time} seconds")
# Verify that the timeout happened within the expected time window
# It should be close to timeout_value, not timeout_value * num_retries
assert elapsed_time < max_allowed_time, (
f"Timeout took too long! Expected ~{timeout_value}s, "
f"got {elapsed_time:.2f}s. This suggests timeout is being "
f"applied per-retry instead of to the total operation."
)
# Also verify it's not TOO fast (sanity check)
assert elapsed_time >= timeout_value * 0.5, (
f"Timeout happened too quickly: {elapsed_time:.2f}s. "
f"Expected at least {timeout_value * 0.5}s"
)
print("✓ Timeout correctly applied to total operation time, not per-retry")
except Exception as e:
pytest.fail(
f"Expected timeout error but got different error: {type(e).__name__}: {e}"
)
@pytest.mark.asyncio
async def test_timeout_with_retries_disabled():
"""
Test that timeout works correctly when retries are explicitly disabled.
This should timeout even faster since there are no retry attempts.
"""
litellm.set_verbose = False
timeout_value = 2.0
max_allowed_time = timeout_value + 0.5 # Even tighter bound with no retries
start_time = time.time()
try:
response = await litellm.acompletion(
model="gpt-3.5-turbo",
timeout=timeout_value,
max_retries=0, # Disable retries
messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}],
)
pytest.fail("Expected timeout error but got a response")
except (openai.APITimeoutError, litellm.exceptions.Timeout) as e:
elapsed_time = time.time() - start_time
print(f"Timeout with no retries occurred after {elapsed_time:.2f} seconds")
assert elapsed_time < max_allowed_time, (
f"Timeout took too long even with retries disabled! "
f"Expected ~{timeout_value}s, got {elapsed_time:.2f}s"
)
print("✓ Timeout works correctly with retries disabled")