fix(openai_like): honor the deployment timeout and stream_timeout on async streams

This commit is contained in:
mateo-berri 2026-11-07 16:41:06 -08:00
parent 7617e66acf
commit 32295c885e
6 changed files with 136 additions and 7 deletions

View file

@ -1,6 +1,7 @@
import json
from typing import Final
import httpx
from pydantic import ConfigDict, TypeAdapter
import litellm
@ -144,6 +145,8 @@ class ModelResponseIterator:
async for raw_chunk in self.async_response_iterator:
if sse_data := _sse_data(raw_chunk):
return sse_data
except httpx.TimeoutException:
raise
except Exception as e:
raise RuntimeError(f"Error receiving chunk from stream: {e}")
raise StopAsyncIteration

View file

@ -54,11 +54,12 @@ async def make_call(
logging_obj,
streaming_decoder: CustomStreamingDecoder | None = None,
fake_stream: bool = False,
timeout: float | httpx.Timeout | None = None,
):
if client is None:
client = litellm.module_level_aclient
response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream)
response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream, timeout=timeout)
if streaming_decoder is not None:
completion_stream = streaming_decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024))
@ -146,6 +147,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
client: AsyncHTTPHandler | None = None,
streaming_decoder: CustomStreamingDecoder | None = None,
fake_stream: bool = False,
timeout: float | httpx.Timeout | None = None,
) -> CustomStreamWrapper:
data["stream"] = True
completion_stream: Final = await make_call(
@ -157,6 +159,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
messages=messages,
logging_obj=logging_obj,
streaming_decoder=streaming_decoder,
timeout=timeout,
)
streamwrapper: Final = CustomStreamWrapper(
completion_stream=completion_stream,
@ -204,8 +207,8 @@ class OpenAILikeChatHandler(OpenAILikeBase):
status_code=e.response.status_code,
message=e.response.text,
)
except httpx.TimeoutException:
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
except litellm.Timeout as e:
raise OpenAILikeError(status_code=408, message=str(e))
except Exception as e:
raise OpenAILikeError(status_code=500, message=str(e))
@ -319,6 +322,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
custom_llm_provider=custom_llm_provider,
streaming_decoder=streaming_decoder,
fake_stream=fake_stream,
timeout=timeout,
)
else:
return self.acompletion_function(
@ -377,8 +381,8 @@ class OpenAILikeChatHandler(OpenAILikeBase):
status_code=e.response.status_code,
message=e.response.text,
)
except httpx.TimeoutException:
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
except litellm.Timeout as e:
raise OpenAILikeError(status_code=408, message=str(e))
except Exception as e:
raise OpenAILikeError(status_code=500, message=str(e))
return OpenAILikeChatConfig._transform_response(

View file

@ -3317,6 +3317,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes
model: Final = ctx.model
model_response: Final = ctx.model_response
optional_params: Final = ctx.optional_params
timeout: Final = ctx.timeout
maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key
@ -3334,6 +3335,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes
encoding=_get_encoding(),
api_key=maritalk_key,
logging_obj=logging,
timeout=timeout,
custom_llm_provider="maritalk",
custom_prompt_dict=custom_prompt_dict,
)

View file

@ -237,7 +237,7 @@ class CompletionDispatchContext:
stream: bool | None
temperature: float | None
text_completion: bool
timeout: float | str | httpx.Timeout | None
timeout: float | httpx.Timeout | None
timeout_for_optional_params: Callable[[Mapping[str, object]], float | httpx.Timeout]
top_p: float | None

View file

@ -1,4 +1,5 @@
import json
from datetime import datetime
from typing import Final
from unittest.mock import Mock
@ -6,8 +7,13 @@ from unittest.mock import Mock
import httpx
import pytest
import litellm
from litellm.exceptions import MidStreamFallbackError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.openai_like.chat.handler import make_call, make_sync_call
from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler, make_call, make_sync_call
from litellm.llms.openai_like.common_utils import OpenAILikeError
from litellm.types.utils import ModelResponse
_FAKE_STREAMED_REPLY: Final = {
"id": "chatcmpl-fake-stream",
@ -106,3 +112,97 @@ async def test_stream_keeps_unicode_line_separators_inside_the_delta(sync_stream
chunks: Final = await _streamed_chunks(sync_stream)
assert "".join(str(chunk["text"]) for chunk in chunks) == _LINE_SEPARATED_TEXT
def _logging_obj(stream: bool) -> LiteLLMLogging:
logging_obj: Final = LiteLLMLogging(
model="mistral-medium-3",
messages=[{"role": "user", "content": "hi"}],
stream=stream,
call_type="acompletion",
start_time=datetime(2026, 1, 1),
litellm_call_id="openai-like-timeout",
function_id="openai-like-timeout",
)
logging_obj.update_environment_variables(
model="mistral-medium-3", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai"
)
return logging_obj
def _complete(client: AsyncHTTPHandler | HTTPHandler, *, acompletion: bool, stream: bool, timeout: float | None):
return OpenAILikeChatHandler().completion(
model="mistral-medium-3",
messages=[{"role": "user", "content": "hi"}],
api_base="https://openai-like.example/v1",
custom_llm_provider="openai",
custom_prompt_dict={},
model_response=ModelResponse(),
print_verbose=print,
encoding=None,
api_key="sk-openai-like",
logging_obj=_logging_obj(stream),
optional_params={"stream": stream},
acompletion=acompletion,
timeout=timeout,
client=client,
)
@pytest.mark.asyncio
async def test_async_stream_sends_the_resolved_timeout_to_the_upstream_request() -> None:
read_timeouts: Final[list[object]] = []
def respond(request: httpx.Request) -> httpx.Response:
read_timeouts.append(request.extensions["timeout"]["read"])
return _streamed_reply(request)
stream: Final = await _complete(
AsyncHTTPHandler(transport=httpx.MockTransport(respond)), acompletion=True, stream=True, timeout=8.0
)
chunks: Final = [chunk async for chunk in stream]
assert read_timeouts == [8.0]
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _LINE_SEPARATED_TEXT
class _StallsAfterFirstChunk(httpx.AsyncByteStream):
async def __aiter__(self):
chunk: Final = {"id": "chatcmpl-stall", "choices": [{"index": 0, "delta": {"content": "partial"}}]}
yield f"data: {json.dumps(chunk)}\n\n".encode()
raise httpx.ReadTimeout("Timeout on reading data from socket")
@pytest.mark.asyncio
async def test_async_stream_read_timeout_surfaces_as_a_408_timeout_the_router_can_fall_back_on() -> None:
transport: Final = httpx.MockTransport(lambda request: httpx.Response(200, stream=_StallsAfterFirstChunk()))
stream: Final = await _complete(AsyncHTTPHandler(transport=transport), acompletion=True, stream=True, timeout=8.0)
with pytest.raises(MidStreamFallbackError) as raised:
[chunk async for chunk in stream]
assert raised.value.status_code == 408
assert isinstance(raised.value.original_exception, litellm.Timeout)
def _read_timeout(request: httpx.Request) -> httpx.Response:
raise httpx.ReadTimeout("Read timed out", request=request)
@pytest.mark.asyncio
async def test_async_non_stream_read_timeout_maps_to_a_408() -> None:
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_read_timeout))
with pytest.raises(OpenAILikeError) as raised:
await _complete(client, acompletion=True, stream=False, timeout=8.0)
assert raised.value.status_code == 408
def test_sync_non_stream_read_timeout_maps_to_a_408() -> None:
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_read_timeout)))
with pytest.raises(OpenAILikeError) as raised:
_complete(client, acompletion=False, stream=False, timeout=8.0)
assert raised.value.status_code == 408

View file

@ -3009,6 +3009,26 @@ def test_responses_bridge_fake_stream_is_bounded_by_the_request_timeout_not_the_
assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "hi"
def test_maritalk_stream_sends_the_stream_timeout_to_the_upstream_request(respx_mock: respx.MockRouter) -> None:
chunk: Final = {"id": "chatcmpl-maritalk", "choices": [{"index": 0, "delta": {"content": "oi"}}]}
route: Final = respx_mock.post("https://maritalk.example/api/chat/completions").respond(
200, content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()
)
response: Final = litellm.completion(
model="maritalk/sabia-3",
messages=[{"role": "user", "content": "hi"}],
stream=True,
stream_timeout=8.0,
timeout=600.0,
api_key="fake-key",
api_base="https://maritalk.example/api",
)
assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "oi"
assert route.calls[0].request.extensions["timeout"]["read"] == 8.0
_FOUNDRY_RESPONSES_FUNCTION_CALL_BODY: Final = {
"id": "resp_foundry",
"object": "response",