mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
fix: forward anthropic messages request timeout
This commit is contained in:
parent
ebd335da67
commit
ccb5473384
2 changed files with 78 additions and 0 deletions
|
|
@ -180,6 +180,37 @@ def _google_genai_streaming_hidden_params(
|
|||
|
||||
|
||||
class BaseLLMHTTPHandler:
|
||||
@staticmethod
|
||||
def _coerce_http_timeout(
|
||||
timeout: Optional[Union[float, int, str, httpx.Timeout]],
|
||||
) -> Optional[Union[float, httpx.Timeout]]:
|
||||
if timeout is None:
|
||||
return None
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
return timeout
|
||||
if isinstance(timeout, str):
|
||||
if timeout.startswith("os.environ/"):
|
||||
timeout = litellm.get_secret(timeout) # type: ignore[assignment]
|
||||
if timeout is None:
|
||||
return None
|
||||
return float(timeout)
|
||||
return float(timeout)
|
||||
|
||||
@staticmethod
|
||||
def _get_anthropic_messages_timeout(
|
||||
*,
|
||||
litellm_params: GenericLiteLLMParams,
|
||||
stream: bool,
|
||||
) -> Optional[Union[float, httpx.Timeout]]:
|
||||
request_timeout = dict(litellm_params).get("request_timeout")
|
||||
if stream and litellm_params.stream_timeout is not None:
|
||||
return BaseLLMHTTPHandler._coerce_http_timeout(
|
||||
litellm_params.stream_timeout
|
||||
)
|
||||
if litellm_params.timeout is not None:
|
||||
return BaseLLMHTTPHandler._coerce_http_timeout(litellm_params.timeout)
|
||||
return BaseLLMHTTPHandler._coerce_http_timeout(request_timeout)
|
||||
|
||||
async def _make_common_async_call(
|
||||
self,
|
||||
async_httpx_client: AsyncHTTPHandler,
|
||||
|
|
@ -1865,6 +1896,10 @@ class BaseLLMHTTPHandler:
|
|||
)
|
||||
litellm_params_dict = dict(litellm_params)
|
||||
optional_params_dict = dict(litellm_params)
|
||||
timeout = self._get_anthropic_messages_timeout(
|
||||
litellm_params=litellm_params,
|
||||
stream=stream,
|
||||
)
|
||||
for attempt_idx in range(max_attempts):
|
||||
try:
|
||||
response = await async_httpx_client.post(
|
||||
|
|
@ -1873,6 +1908,7 @@ class BaseLLMHTTPHandler:
|
|||
data=signed_json_body or json.dumps(request_body),
|
||||
stream=stream or False,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -111,6 +111,48 @@ def test_fingerprint_agentic_tools_is_deterministic():
|
|||
) == handler._fingerprint_agentic_tools(tools_b)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"litellm_params,stream,expected_timeout",
|
||||
[
|
||||
(GenericLiteLLMParams(timeout=1.25), False, 1.25),
|
||||
(GenericLiteLLMParams(timeout=30, stream_timeout=0.75), True, 0.75),
|
||||
(GenericLiteLLMParams(request_timeout=2.5), False, 2.5),
|
||||
],
|
||||
)
|
||||
async def test_anthropic_messages_post_forwards_request_timeout(
|
||||
litellm_params, stream, expected_timeout
|
||||
):
|
||||
"""Anthropic /v1/messages must honor per-request timeout settings."""
|
||||
|
||||
handler = BaseLLMHTTPHandler()
|
||||
mock_client = AsyncMock()
|
||||
mock_response = Mock()
|
||||
mock_response.raise_for_status = Mock()
|
||||
mock_client.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
mock_config = Mock()
|
||||
mock_config.max_retry_on_anthropic_messages_http_error = 1
|
||||
|
||||
response = await handler._async_post_anthropic_messages_with_http_error_retry(
|
||||
async_httpx_client=mock_client,
|
||||
request_url="https://api.anthropic.com/v1/messages",
|
||||
headers={"x-api-key": "test-key"},
|
||||
signed_json_body=None,
|
||||
request_body={"model": "claude-3-5-sonnet", "messages": []},
|
||||
stream=stream,
|
||||
logging_obj=Mock(),
|
||||
provider_config=mock_config,
|
||||
litellm_params=litellm_params,
|
||||
api_key="test-key",
|
||||
model="claude-3-5-sonnet",
|
||||
)
|
||||
|
||||
assert response == mock_response
|
||||
mock_client.post.assert_awaited_once()
|
||||
assert mock_client.post.call_args.kwargs["timeout"] == expected_timeout
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_anthropic_messages_handler_extra_headers():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue