mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(openai_like): honor the deployment timeout and stream_timeout on async streams
This commit is contained in:
parent
7617e66acf
commit
32295c885e
6 changed files with 136 additions and 7 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
|
|
@ -144,6 +145,8 @@ class ModelResponseIterator:
|
|||
async for raw_chunk in self.async_response_iterator:
|
||||
if sse_data := _sse_data(raw_chunk):
|
||||
return sse_data
|
||||
except httpx.TimeoutException:
|
||||
raise
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Error receiving chunk from stream: {e}")
|
||||
raise StopAsyncIteration
|
||||
|
|
|
|||
|
|
@ -54,11 +54,12 @@ async def make_call(
|
|||
logging_obj,
|
||||
streaming_decoder: CustomStreamingDecoder | None = None,
|
||||
fake_stream: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
):
|
||||
if client is None:
|
||||
client = litellm.module_level_aclient
|
||||
|
||||
response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream)
|
||||
response: Final = await client.post(api_base, headers=headers, data=data, stream=not fake_stream, timeout=timeout)
|
||||
|
||||
if streaming_decoder is not None:
|
||||
completion_stream = streaming_decoder.aiter_bytes(response.aiter_bytes(chunk_size=1024))
|
||||
|
|
@ -146,6 +147,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
|
|||
client: AsyncHTTPHandler | None = None,
|
||||
streaming_decoder: CustomStreamingDecoder | None = None,
|
||||
fake_stream: bool = False,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
) -> CustomStreamWrapper:
|
||||
data["stream"] = True
|
||||
completion_stream: Final = await make_call(
|
||||
|
|
@ -157,6 +159,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
|
|||
messages=messages,
|
||||
logging_obj=logging_obj,
|
||||
streaming_decoder=streaming_decoder,
|
||||
timeout=timeout,
|
||||
)
|
||||
streamwrapper: Final = CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -204,8 +207,8 @@ class OpenAILikeChatHandler(OpenAILikeBase):
|
|||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
|
||||
except litellm.Timeout as e:
|
||||
raise OpenAILikeError(status_code=408, message=str(e))
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
|
||||
|
|
@ -319,6 +322,7 @@ class OpenAILikeChatHandler(OpenAILikeBase):
|
|||
custom_llm_provider=custom_llm_provider,
|
||||
streaming_decoder=streaming_decoder,
|
||||
fake_stream=fake_stream,
|
||||
timeout=timeout,
|
||||
)
|
||||
else:
|
||||
return self.acompletion_function(
|
||||
|
|
@ -377,8 +381,8 @@ class OpenAILikeChatHandler(OpenAILikeBase):
|
|||
status_code=e.response.status_code,
|
||||
message=e.response.text,
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise OpenAILikeError(status_code=408, message="Timeout error occurred.")
|
||||
except litellm.Timeout as e:
|
||||
raise OpenAILikeError(status_code=408, message=str(e))
|
||||
except Exception as e:
|
||||
raise OpenAILikeError(status_code=500, message=str(e))
|
||||
return OpenAILikeChatConfig._transform_response(
|
||||
|
|
|
|||
|
|
@ -3317,6 +3317,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
model: Final = ctx.model
|
||||
model_response: Final = ctx.model_response
|
||||
optional_params: Final = ctx.optional_params
|
||||
timeout: Final = ctx.timeout
|
||||
|
||||
maritalk_key: Final = api_key or litellm.maritalk_key or get_secret("MARITALK_API_KEY") or litellm.api_key
|
||||
|
||||
|
|
@ -3334,6 +3335,7 @@ def _complete_maritalk(ctx: CompletionDispatchContext) -> _CompletionDispatchRes
|
|||
encoding=_get_encoding(),
|
||||
api_key=maritalk_key,
|
||||
logging_obj=logging,
|
||||
timeout=timeout,
|
||||
custom_llm_provider="maritalk",
|
||||
custom_prompt_dict=custom_prompt_dict,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -237,7 +237,7 @@ class CompletionDispatchContext:
|
|||
stream: bool | None
|
||||
temperature: float | None
|
||||
text_completion: bool
|
||||
timeout: float | str | httpx.Timeout | None
|
||||
timeout: float | httpx.Timeout | None
|
||||
timeout_for_optional_params: Callable[[Mapping[str, object]], float | httpx.Timeout]
|
||||
top_p: float | None
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
|
@ -6,8 +7,13 @@ from unittest.mock import Mock
|
|||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.exceptions import MidStreamFallbackError
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.openai_like.chat.handler import make_call, make_sync_call
|
||||
from litellm.llms.openai_like.chat.handler import OpenAILikeChatHandler, make_call, make_sync_call
|
||||
from litellm.llms.openai_like.common_utils import OpenAILikeError
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
_FAKE_STREAMED_REPLY: Final = {
|
||||
"id": "chatcmpl-fake-stream",
|
||||
|
|
@ -106,3 +112,97 @@ async def test_stream_keeps_unicode_line_separators_inside_the_delta(sync_stream
|
|||
chunks: Final = await _streamed_chunks(sync_stream)
|
||||
|
||||
assert "".join(str(chunk["text"]) for chunk in chunks) == _LINE_SEPARATED_TEXT
|
||||
|
||||
|
||||
def _logging_obj(stream: bool) -> LiteLLMLogging:
|
||||
logging_obj: Final = LiteLLMLogging(
|
||||
model="mistral-medium-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=stream,
|
||||
call_type="acompletion",
|
||||
start_time=datetime(2026, 1, 1),
|
||||
litellm_call_id="openai-like-timeout",
|
||||
function_id="openai-like-timeout",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
model="mistral-medium-3", user=None, optional_params={}, litellm_params={}, custom_llm_provider="openai"
|
||||
)
|
||||
return logging_obj
|
||||
|
||||
|
||||
def _complete(client: AsyncHTTPHandler | HTTPHandler, *, acompletion: bool, stream: bool, timeout: float | None):
|
||||
return OpenAILikeChatHandler().completion(
|
||||
model="mistral-medium-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://openai-like.example/v1",
|
||||
custom_llm_provider="openai",
|
||||
custom_prompt_dict={},
|
||||
model_response=ModelResponse(),
|
||||
print_verbose=print,
|
||||
encoding=None,
|
||||
api_key="sk-openai-like",
|
||||
logging_obj=_logging_obj(stream),
|
||||
optional_params={"stream": stream},
|
||||
acompletion=acompletion,
|
||||
timeout=timeout,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_stream_sends_the_resolved_timeout_to_the_upstream_request() -> None:
|
||||
read_timeouts: Final[list[object]] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
read_timeouts.append(request.extensions["timeout"]["read"])
|
||||
return _streamed_reply(request)
|
||||
|
||||
stream: Final = await _complete(
|
||||
AsyncHTTPHandler(transport=httpx.MockTransport(respond)), acompletion=True, stream=True, timeout=8.0
|
||||
)
|
||||
chunks: Final = [chunk async for chunk in stream]
|
||||
|
||||
assert read_timeouts == [8.0]
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _LINE_SEPARATED_TEXT
|
||||
|
||||
|
||||
class _StallsAfterFirstChunk(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
chunk: Final = {"id": "chatcmpl-stall", "choices": [{"index": 0, "delta": {"content": "partial"}}]}
|
||||
yield f"data: {json.dumps(chunk)}\n\n".encode()
|
||||
raise httpx.ReadTimeout("Timeout on reading data from socket")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_stream_read_timeout_surfaces_as_a_408_timeout_the_router_can_fall_back_on() -> None:
|
||||
transport: Final = httpx.MockTransport(lambda request: httpx.Response(200, stream=_StallsAfterFirstChunk()))
|
||||
stream: Final = await _complete(AsyncHTTPHandler(transport=transport), acompletion=True, stream=True, timeout=8.0)
|
||||
|
||||
with pytest.raises(MidStreamFallbackError) as raised:
|
||||
[chunk async for chunk in stream]
|
||||
|
||||
assert raised.value.status_code == 408
|
||||
assert isinstance(raised.value.original_exception, litellm.Timeout)
|
||||
|
||||
|
||||
def _read_timeout(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ReadTimeout("Read timed out", request=request)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_non_stream_read_timeout_maps_to_a_408() -> None:
|
||||
client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_read_timeout))
|
||||
|
||||
with pytest.raises(OpenAILikeError) as raised:
|
||||
await _complete(client, acompletion=True, stream=False, timeout=8.0)
|
||||
|
||||
assert raised.value.status_code == 408
|
||||
|
||||
|
||||
def test_sync_non_stream_read_timeout_maps_to_a_408() -> None:
|
||||
client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_read_timeout)))
|
||||
|
||||
with pytest.raises(OpenAILikeError) as raised:
|
||||
_complete(client, acompletion=False, stream=False, timeout=8.0)
|
||||
|
||||
assert raised.value.status_code == 408
|
||||
|
|
|
|||
|
|
@ -3009,6 +3009,26 @@ def test_responses_bridge_fake_stream_is_bounded_by_the_request_timeout_not_the_
|
|||
assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "hi"
|
||||
|
||||
|
||||
def test_maritalk_stream_sends_the_stream_timeout_to_the_upstream_request(respx_mock: respx.MockRouter) -> None:
|
||||
chunk: Final = {"id": "chatcmpl-maritalk", "choices": [{"index": 0, "delta": {"content": "oi"}}]}
|
||||
route: Final = respx_mock.post("https://maritalk.example/api/chat/completions").respond(
|
||||
200, content=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n".encode()
|
||||
)
|
||||
|
||||
response: Final = litellm.completion(
|
||||
model="maritalk/sabia-3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
stream_timeout=8.0,
|
||||
timeout=600.0,
|
||||
api_key="fake-key",
|
||||
api_base="https://maritalk.example/api",
|
||||
)
|
||||
|
||||
assert "".join(chunk.choices[0].delta.content or "" for chunk in response) == "oi"
|
||||
assert route.calls[0].request.extensions["timeout"]["read"] == 8.0
|
||||
|
||||
|
||||
_FOUNDRY_RESPONSES_FUNCTION_CALL_BODY: Final = {
|
||||
"id": "resp_foundry",
|
||||
"object": "response",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue