diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 96d0ad48b79..b47fc50e196 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1879,6 +1879,7 @@ class BaseLLMHTTPHandler: litellm_params: GenericLiteLLMParams, api_key: Optional[str], model: str, + timeout: Optional[Union[float, httpx.Timeout]] = None, ) -> httpx.Response: max_attempts = max(provider_config.max_retry_on_anthropic_messages_http_error, 1) litellm_params_dict = dict(litellm_params) @@ -1891,6 +1892,7 @@ class BaseLLMHTTPHandler: data=signed_json_body or json.dumps(request_body), stream=stream or False, logging_obj=logging_obj, + timeout=timeout, ) response.raise_for_status() return response @@ -1925,6 +1927,32 @@ class BaseLLMHTTPHandler: raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return") + @staticmethod + def _resolve_anthropic_messages_timeout( + litellm_params: GenericLiteLLMParams, + stream: bool, + custom_llm_provider: str, + ) -> Optional[Union[float, httpx.Timeout]]: + from litellm.litellm_core_utils.completion_timeout import CompletionTimeout + from litellm.litellm_core_utils.request_timeout_resolver import ( + get_configured_request_timeout, + ) + from litellm.utils import supports_httpx_timeout + + stream_timeout = litellm_params.get("stream_timeout") if stream else None + model_timeout = stream_timeout if stream_timeout is not None else litellm_params.get("timeout") + request_timeout = litellm_params.get("request_timeout") + global_timeout = get_configured_request_timeout() + if model_timeout is None and request_timeout is None and global_timeout is None: + return None + return CompletionTimeout.resolve( + model_timeout, + {"request_timeout": request_timeout}, + custom_llm_provider, + global_timeout=global_timeout, + supports_httpx_timeout=supports_httpx_timeout, + ) + async def async_anthropic_messages_handler( self, model: str, @@ -2075,6 +2103,11 @@ class BaseLLMHTTPHandler: litellm_params=litellm_params, api_key=api_key, model=model, + timeout=self._resolve_anthropic_messages_timeout( + litellm_params=litellm_params, + stream=stream or False, + custom_llm_provider=custom_llm_provider, + ), ) # used for logging + cost tracking diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py index 926f40a6c67..fddd8d09dfc 100644 --- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py @@ -1084,6 +1084,147 @@ def test_sync_delete_responses_sets_json_content_type(): # --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "litellm_params_kwargs, stream, global_timeout, expected", + [ + ({"timeout": 12.0}, False, None, 12.0), + ({"request_timeout": 30.0}, False, None, 30.0), + ({}, False, 1500.0, 1500.0), + ({"timeout": 5.0, "stream_timeout": 50.0}, True, None, 50.0), + ({"timeout": 5.0, "stream_timeout": 50.0}, False, None, 5.0), + ({"timeout": 5.0, "request_timeout": 30.0}, False, None, 5.0), + ({}, False, None, None), + ({}, True, None, None), + ], +) +def test_resolve_anthropic_messages_timeout( + monkeypatch, litellm_params_kwargs, stream, global_timeout, expected +): + from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS + + if global_timeout is None: + monkeypatch.setattr( + "litellm.request_timeout", + float(DEFAULT_REQUEST_TIMEOUT_SECONDS), + raising=False, + ) + monkeypatch.setattr( + "litellm.request_timeout_explicitly_set", + False, + raising=False, + ) + else: + monkeypatch.setattr("litellm.request_timeout", global_timeout, raising=False) + monkeypatch.setattr( + "litellm.request_timeout_explicitly_set", True, raising=False + ) + + resolved = BaseLLMHTTPHandler._resolve_anthropic_messages_timeout( + litellm_params=GenericLiteLLMParams(**litellm_params_kwargs), + stream=stream, + custom_llm_provider="anthropic", + ) + + assert resolved == expected + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeypatch): + from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "request_timeout", float(DEFAULT_REQUEST_TIMEOUT_SECONDS)) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False) + handler = BaseLLMHTTPHandler() + + mock_config = Mock() + mock_config.validate_anthropic_messages_environment = Mock( + return_value=({"x-api-key": "k"}, "https://api.anthropic.com") + ) + mock_config.should_filter_anthropic_beta_headers = Mock(return_value=False) + mock_config.transform_anthropic_messages_request = Mock( + return_value={"model": "claude", "messages": []} + ) + mock_config.get_complete_url = Mock(return_value="https://api.anthropic.com/v1/messages") + mock_config.sign_request = Mock(return_value=({"x-api-key": "k"}, None)) + mock_config.max_retry_on_anthropic_messages_http_error = 1 + expected_response = {"id": "msg_1", "content": []} + mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response) + + ok_response = Mock() + ok_response.raise_for_status = Mock(return_value=None) + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=ok_response) + + logging_obj = Mock() + logging_obj.model_call_details = {} + logging_obj.dynamic_success_callbacks = [] + + result = await handler.async_anthropic_messages_handler( + model="claude", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=mock_config, + anthropic_messages_optional_request_params={}, + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(request_timeout=0.3), + logging_obj=logging_obj, + client=mock_client, + kwargs={}, + ) + + assert result is expected_response + assert mock_client.post.await_args.kwargs["timeout"] == 0.3 + + +@pytest.mark.asyncio +async def test_async_anthropic_messages_handler_forwards_stream_timeout(monkeypatch): + from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "request_timeout", float(DEFAULT_REQUEST_TIMEOUT_SECONDS)) + monkeypatch.setattr(litellm, "request_timeout_explicitly_set", False) + handler = BaseLLMHTTPHandler() + + mock_config = Mock() + mock_config.validate_anthropic_messages_environment = Mock( + return_value=({"x-api-key": "k"}, "https://api.anthropic.com") + ) + mock_config.should_filter_anthropic_beta_headers = Mock(return_value=False) + mock_config.transform_anthropic_messages_request = Mock( + return_value={"model": "claude", "messages": []} + ) + mock_config.get_complete_url = Mock(return_value="https://api.anthropic.com/v1/messages") + mock_config.sign_request = Mock(return_value=({"x-api-key": "k"}, None)) + mock_config.max_retry_on_anthropic_messages_http_error = 1 + mock_config.get_async_streaming_response_iterator = Mock(return_value=Mock()) + + ok_response = Mock() + ok_response.raise_for_status = Mock(return_value=None) + ok_response.headers = httpx.Headers({}) + mock_client = AsyncMock(spec=AsyncHTTPHandler) + mock_client.post = AsyncMock(return_value=ok_response) + + logging_obj = Mock() + logging_obj.model_call_details = {} + logging_obj.dynamic_success_callbacks = [] + + await handler.async_anthropic_messages_handler( + model="claude", + messages=[{"role": "user", "content": "hi"}], + anthropic_messages_provider_config=mock_config, + anthropic_messages_optional_request_params={}, + custom_llm_provider="anthropic", + litellm_params=GenericLiteLLMParams(timeout=9.0, stream_timeout=0.7), + logging_obj=logging_obj, + client=mock_client, + stream=True, + kwargs={}, + ) + + assert mock_client.post.await_args.kwargs["stream"] is True + assert mock_client.post.await_args.kwargs["timeout"] == 0.7 + + @pytest.mark.asyncio async def test_anthropic_post_uses_prebuilt_body_without_redumping(): """When the caller passes a pre-serialized (unsigned) body, attempt 0 must @@ -1894,7 +2035,9 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques ok_response = httpx.Response(200, json={"id": "msg_1"}, request=httpx.Request("POST", request_url)) class FakeAsyncClient: - async def post(self, url, headers, data, stream=False, logging_obj=None): + async def post( + self, url, headers, data, stream=False, logging_obj=None, timeout=None + ): posts.append({"headers": dict(headers), "data": data}) return invalid_signature_response if len(posts) == 1 else ok_response