diff --git a/litellm/main.py b/litellm/main.py index 5b8c569a390..23922e3c8bf 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -599,15 +599,8 @@ async def acompletion( # noqa: PLR0915 # Add the context to the function ctx = contextvars.copy_context() func_with_context = partial(ctx.run, func) - - if timeout is not None and isinstance(timeout, (int, float)): - timeout_value = float(timeout) - init_response = await asyncio.wait_for( - loop.run_in_executor(None, func_with_context), timeout=timeout_value - ) - else: - init_response = await loop.run_in_executor(None, func_with_context) - + + init_response = await loop.run_in_executor(None, func_with_context) if isinstance(init_response, dict) or isinstance( init_response, ModelResponse ): ## CACHING SCENARIO @@ -615,11 +608,7 @@ async def acompletion( # noqa: PLR0915 response = ModelResponse(**init_response) response = init_response elif asyncio.iscoroutine(init_response): - if timeout is not None and isinstance(timeout, (int, float)): - timeout_value = float(timeout) - response = await asyncio.wait_for(init_response, timeout=timeout_value) - else: - response = await init_response + response = await init_response else: response = init_response # type: ignore @@ -636,15 +625,6 @@ async def acompletion( # noqa: PLR0915 loop=loop ) # sets the logging event loop if the user does sync streaming (e.g. on proxy for sagemaker calls) return response - except asyncio.TimeoutError: - custom_llm_provider = custom_llm_provider or "openai" - from litellm.exceptions import Timeout - - raise Timeout( - message=f"Request timed out after {timeout} seconds", - model=model, - llm_provider=custom_llm_provider, - ) except Exception as e: custom_llm_provider = custom_llm_provider or "openai" raise exception_type( diff --git a/tests/local_testing/test_timeout.py b/tests/local_testing/test_timeout.py index 622c47644a7..7f9837e81dd 100644 --- a/tests/local_testing/test_timeout.py +++ b/tests/local_testing/test_timeout.py @@ -96,7 +96,6 @@ def test_bedrock_timeout(): def test_hanging_request_azure(): litellm.set_verbose = True import asyncio - from unittest.mock import AsyncMock, patch, MagicMock try: router = litellm.Router( @@ -105,8 +104,8 @@ def test_hanging_request_azure(): "model_name": "azure-gpt", "litellm_params": { "model": "azure/gpt-4o-new-test", - "api_base": os.environ.get("AZURE_API_BASE", "https://test.openai.azure.com"), - "api_key": os.environ.get("AZURE_API_KEY", "test-key"), + "api_base": os.environ["AZURE_API_BASE"], + "api_key": os.environ["AZURE_API_KEY"], }, }, { @@ -120,25 +119,16 @@ def test_hanging_request_azure(): encoded = litellm.utils.encode(model="gpt-3.5-turbo", text="blue")[0] async def _test(): - # Mock the Azure OpenAI client's create method to simulate a hanging request - with patch("openai.resources.chat.completions.AsyncCompletions.create") as mock_create: - # Simulate a hanging request that takes longer than the timeout - async def hanging_request(*args, **kwargs): - await asyncio.sleep(10) # Sleep much longer than the 0.01s timeout - return MagicMock() - - mock_create.side_effect = hanging_request - - response = await router.acompletion( - model="azure-gpt", - messages=[ - {"role": "user", "content": f"what color is red {uuid.uuid4()}"} - ], - logit_bias={encoded: 100}, - timeout=0.01, - ) - print(response) - return response + response = await router.acompletion( + model="azure-gpt", + messages=[ + {"role": "user", "content": f"what color is red {uuid.uuid4()}"} + ], + logit_bias={encoded: 100}, + timeout=0.01, + ) + print(response) + return response response = asyncio.run(_test()) @@ -150,16 +140,9 @@ def test_hanging_request_azure(): ) print(type(e)) pass - except litellm.exceptions.APIError as e: - # Azure may convert CancelledError to APIError - this is also acceptable for timeout scenarios - print( - "Passed: Raised APIError due to timeout (CancelledError). This is acceptable.", e - ) - print(type(e)) - pass except Exception as e: pytest.fail( - f"Did not raise error `openai.APITimeoutError` or `litellm.exceptions.APIError`. Instead raised error type: {type(e)}, Error: {e}" + f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}" ) @@ -302,94 +285,3 @@ async def test_anthropic_timeout(streaming, sync_mode): ) print(type(e)) pass - - -@pytest.mark.asyncio -async def test_timeout_respects_total_time_not_per_retry(): - """ - Test that timeout applies to the TOTAL operation time, not per-retry. - - This test ensures that when a user sets timeout=2, the entire operation - (including all retries) times out at ~2 seconds, not at 2s * num_retries. - - This is a regression test for the issue where timeout was being applied - per-retry attempt, causing the total time to be much longer than expected. - """ - litellm.set_verbose = False - - timeout_value = 2.0 - # Allow for some overhead (network, processing, etc.) - # but ensure we don't wait for multiple retries - max_allowed_time = timeout_value + 1.0 # 3 seconds max - - start_time = time.time() - - try: - # This should timeout because we're asking for a long response - # with a very short timeout - response = await litellm.acompletion( - model="gpt-3.5-turbo", - timeout=timeout_value, - messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}], - ) - pytest.fail("Expected timeout error but got a response") - except (openai.APITimeoutError, litellm.exceptions.Timeout) as e: - elapsed_time = time.time() - start_time - - print(f"Timeout occurred after {elapsed_time:.2f} seconds") - print(f"Expected timeout: {timeout_value} seconds") - print(f"Max allowed time: {max_allowed_time} seconds") - - # Verify that the timeout happened within the expected time window - # It should be close to timeout_value, not timeout_value * num_retries - assert elapsed_time < max_allowed_time, ( - f"Timeout took too long! Expected ~{timeout_value}s, " - f"got {elapsed_time:.2f}s. This suggests timeout is being " - f"applied per-retry instead of to the total operation." - ) - - # Also verify it's not TOO fast (sanity check) - assert elapsed_time >= timeout_value * 0.5, ( - f"Timeout happened too quickly: {elapsed_time:.2f}s. " - f"Expected at least {timeout_value * 0.5}s" - ) - - print("✓ Timeout correctly applied to total operation time, not per-retry") - except Exception as e: - pytest.fail( - f"Expected timeout error but got different error: {type(e).__name__}: {e}" - ) - - -@pytest.mark.asyncio -async def test_timeout_with_retries_disabled(): - """ - Test that timeout works correctly when retries are explicitly disabled. - This should timeout even faster since there are no retry attempts. - """ - litellm.set_verbose = False - - timeout_value = 2.0 - max_allowed_time = timeout_value + 0.5 # Even tighter bound with no retries - - start_time = time.time() - - try: - response = await litellm.acompletion( - model="gpt-3.5-turbo", - timeout=timeout_value, - max_retries=0, # Disable retries - messages=[{"role": "user", "content": "Write a very long detailed essay about the history of computing, at least 5000 words."}], - ) - pytest.fail("Expected timeout error but got a response") - except (openai.APITimeoutError, litellm.exceptions.Timeout) as e: - elapsed_time = time.time() - start_time - - print(f"Timeout with no retries occurred after {elapsed_time:.2f} seconds") - - assert elapsed_time < max_allowed_time, ( - f"Timeout took too long even with retries disabled! " - f"Expected ~{timeout_value}s, got {elapsed_time:.2f}s" - ) - - print("✓ Timeout works correctly with retries disabled")