diff --git a/litellm/llms/databricks/streaming_utils.py b/litellm/llms/databricks/streaming_utils.py index cc85012b803..f90a2c61d36 100644 --- a/litellm/llms/databricks/streaming_utils.py +++ b/litellm/llms/databricks/streaming_utils.py @@ -1,6 +1,7 @@ import json from typing import Final +import httpx from pydantic import ConfigDict, TypeAdapter import litellm @@ -144,6 +145,8 @@ class ModelResponseIterator: async for raw_chunk in self.async_response_iterator: if sse_data := _sse_data(raw_chunk): return sse_data + except httpx.TimeoutException: + raise except Exception as e: raise RuntimeError(f"Error receiving chunk from stream: {e}") raise StopAsyncIteration diff --git a/litellm/llms/openai_like/chat/handler.py b/litellm/llms/openai_like/chat/handler.py index a450947afdf..8f1104ae3b1 100644 --- a/litellm/llms/openai_like/chat/handler.py +++ b/litellm/llms/openai_like/chat/handler.py @@ -54,11 +54,12 @@ async def make_call( logging_obj, streaming_decoder: CustomStreamingDecoder | None = None, fake_stream: bool = False, + timeout: float | httpx.Timeout | None = None, ): if client is None: client = litellm.module_level_aclient - response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream) + response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream, timeout=timeout) if streaming_decoder is not None: completion_stream = streaming_decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024)) @@ -146,6 +147,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): client: AsyncHTTPHandler | None = None, streaming_decoder: CustomStreamingDecoder | None = None, fake_stream: bool = False, + timeout: float | httpx.Timeout | None = None, ) -> CustomStreamWrapper: data["stream"] = True completion_stream: Final = await make_call( @@ -157,6 +159,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): messages=messages, logging_obj=logging_obj, streaming_decoder=streaming_decoder, + timeout=timeout, ) streamwrapper: Final = CustomStreamWrapper( completion_stream=completion_stream, @@ -204,8 +207,8 @@ class OpenAILikeChatHandler(OpenAILikeBase): status_code=e.response.status_code, message=e.response.text, ) - except httpx.TimeoutException: - raise OpenAILikeError(status_code=408, message="Timeout error occurred.") + except litellm.Timeout as e: + raise OpenAILikeError(status_code=408, message=str(e)) except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) @@ -319,6 +322,7 @@ class OpenAILikeChatHandler(OpenAILikeBase): custom_llm_provider=custom_llm_provider, streaming_decoder=streaming_decoder, fake_stream=fake_stream, + timeout=timeout, ) else: return self.acompletion_function( @@ -377,8 +381,8 @@ class OpenAILikeChatHandler(OpenAILikeBase): status_code=e.response.status_code, message=e.response.text, ) - except httpx.TimeoutException: - raise OpenAILikeError(status_code=408, message="Timeout error occurred.") + except litellm.Timeout as e: + raise OpenAILikeError(status_code=408, message=str(e)) except Exception as e: raise OpenAILikeError(status_code=500, message=str(e)) return OpenAILikeChatConfig._transform_response( diff --git a/litellm/main.py b/litellm/main.py index f3620d7a8c2..2a1189962d1 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3317,6 +3317,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes model: Final = ctx.model model_response: Final = ctx.model_response optional_params: Final = ctx.optional_params + timeout: Final = ctx.timeout maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key @@ -3334,6 +3335,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes encoding=_get_encoding(), api_key=maritalk_key, logging_obj=logging, + timeout=timeout, custom_llm_provider="maritalk", custom_prompt_dict=custom_prompt_dict, ) diff --git a/litellm/types/completion.py b/litellm/types/completion.py index 0f4bdbd75ce..dc92addf236 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -237,7 +237,7 @@ class CompletionDispatchContext: stream: bool | None temperature: float | None text_completion: bool - timeout: float | str | httpx.Timeout | None + timeout: float | httpx.Timeout | None timeout_for_optional_params: Callable[[Mapping[str, object]], float | httpx.Timeout] top_p: float | None diff --git a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py index 3df1efe311b..a1aba467dee 100644 --- a/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py +++ b/tests/unit/llms/openai_like/chat/test_openai_like_chat_handler.py @@ -1,4 +1,5 @@ import json +from datetime import datetime from typing import Final from unittest.mock import Mock @@ -6,8 +7,13 @@ from unittest.mock import Mock import httpx import pytest +import litellm +from litellm.exceptions import MidStreamFallbackError +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.llms.openai_like.chat.handler import make_call, make_sync_call +from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler, make_call, make_sync_call +from litellm.llms.openai_like.common_utils import OpenAILikeError +from litellm.types.utils import ModelResponse _FAKE_STREAMED_REPLY: Final = { "id": "chatcmpl-fake-stream", @@ -106,3 +112,97 @@ async def test_stream_keeps_unicode_line_separators_inside_the_delta(sync_stream chunks: Final = await _streamed_chunks(sync_stream) assert "".join(str(chunk["text"]) for chunk in chunks) == _LINE_SEPARATED_TEXT + + +def _logging_obj(stream: bool) -> LiteLLMLogging: + logging_obj: Final = LiteLLMLogging( + model="mistral-medium-3", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="acompletion", + start_time=datetime(2026, 1, 1), + litellm_call_id="openai-like-timeout", + function_id="openai-like-timeout", + ) + logging_obj.update_environment_variables( + model="mistral-medium-3", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai" + ) + return logging_obj + + +def _complete(client: AsyncHTTPHandler | HTTPHandler, *, acompletion: bool, stream: bool, timeout: float | None): + return OpenAILikeChatHandler().completion( + model="mistral-medium-3", + messages=[{"role": "user", "content": "hi"}], + api_base="https://openai-like.example/v1", + custom_llm_provider="openai", + custom_prompt_dict={}, + model_response=ModelResponse(), + print_verbose=print, + encoding=None, + api_key="sk-openai-like", + logging_obj=_logging_obj(stream), + optional_params={"stream": stream}, + acompletion=acompletion, + timeout=timeout, + client=client, + ) + + +@pytest.mark.asyncio +async def test_async_stream_sends_the_resolved_timeout_to_the_upstream_request() -> None: + read_timeouts: Final[list[object]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + read_timeouts.append(request.extensions["timeout"]["read"]) + return _streamed_reply(request) + + stream: Final = await _complete( + AsyncHTTPHandler(transport=httpx.MockTransport(respond)), acompletion=True, stream=True, timeout=8.0 + ) + chunks: Final = [chunk async for chunk in stream] + + assert read_timeouts == [8.0] + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _LINE_SEPARATED_TEXT + + +class _StallsAfterFirstChunk(httpx.AsyncByteStream): + async def __aiter__(self): + chunk: Final = {"id": "chatcmpl-stall", "choices": [{"index": 0, "delta": {"content": "partial"}}]} + yield f"data: {json.dumps(chunk)}\n\n".encode() + raise httpx.ReadTimeout("Timeout on reading data from socket") + + +@pytest.mark.asyncio +async def test_async_stream_read_timeout_surfaces_as_a_408_timeout_the_router_can_fall_back_on() -> None: + transport: Final = httpx.MockTransport(lambda request: httpx.Response(200, stream=_StallsAfterFirstChunk())) + stream: Final = await _complete(AsyncHTTPHandler(transport=transport), acompletion=True, stream=True, timeout=8.0) + + with pytest.raises(MidStreamFallbackError) as raised: + [chunk async for chunk in stream] + + assert raised.value.status_code == 408 + assert isinstance(raised.value.original_exception, litellm.Timeout) + + +def _read_timeout(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout("Read timed out", request=request) + + +@pytest.mark.asyncio +async def test_async_non_stream_read_timeout_maps_to_a_408() -> None: + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_read_timeout)) + + with pytest.raises(OpenAILikeError) as raised: + await _complete(client, acompletion=True, stream=False, timeout=8.0) + + assert raised.value.status_code == 408 + + +def test_sync_non_stream_read_timeout_maps_to_a_408() -> None: + client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_read_timeout))) + + with pytest.raises(OpenAILikeError) as raised: + _complete(client, acompletion=False, stream=False, timeout=8.0) + + assert raised.value.status_code == 408 diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 548f78d148b..7fa1e9f0d02 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -3009,6 +3009,26 @@ def test_responses_bridge_fake_stream_is_bounded_by_the_request_timeout_not_the_ assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "hi" +def test_maritalk_stream_sends_the_stream_timeout_to_the_upstream_request(respx_mock: respx.MockRouter) -> None: + chunk: Final = {"id": "chatcmpl-maritalk", "choices": [{"index": 0, "delta": {"content": "oi"}}]} + route: Final = respx_mock.post("https://maritalk.example/api/chat/completions").respond( + 200, content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode() + ) + + response: Final = litellm.completion( + model="maritalk/sabia-3", + messages=[{"role": "user", "content": "hi"}], + stream=True, + stream_timeout=8.0, + timeout=600.0, + api_key="fake-key", + api_base="https://maritalk.example/api", + ) + + assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "oi" + assert route.calls[0].request.extensions["timeout"]["read"] == 8.0 + + _FOUNDRY_RESPONSES_FUNCTION_CALL_BODY: Final = { "id": "resp_foundry", "object": "response",