From dbfb175303fffa91bf4b371b18da92f19209e508 Mon Sep 17 00:00:00 2001 From: ishaan-j-berri Date: Sat, 24 Oct 2026 10:13:18 -0700 Subject: [PATCH] fix(rerank): preserve HTTP phase timeout limits Keep connection, read, write, and pool limits within the remaining decision-rerank deadline. --- .../handler.py | 18 ++++++- litellm/rerank_api/main.py | 15 +++--- .../test_handler.py | 49 +++++++++++++------ 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/litellm/rerank_api/litellm_decisions_transformation/handler.py b/litellm/rerank_api/litellm_decisions_transformation/handler.py index c8d64d70290..fd8c2a46d40 100644 --- a/litellm/rerank_api/litellm_decisions_transformation/handler.py +++ b/litellm/rerank_api/litellm_decisions_transformation/handler.py @@ -10,6 +10,8 @@ from queue import Empty, SimpleQueue from threading import Event from typing import Final, Literal +import httpx + import litellm from litellm.decisions.call import ( DecisionsCall, @@ -53,6 +55,17 @@ def _call_with_deadline(document: _DocumentCall, deadline: float) -> DecisionsCa remaining: Final = deadline - time.monotonic() if remaining <= 0: return _timeout(document.call.requested_model, document.call.custom_llm_provider) + timeout: Final = document.call.timeout + if isinstance(timeout, httpx.Timeout): + return replace( + document.call, + timeout=httpx.Timeout( + connect=remaining if timeout.connect is None else min(timeout.connect, remaining), + read=remaining if timeout.read is None else min(timeout.read, remaining), + write=remaining if timeout.write is None else min(timeout.write, remaining), + pool=remaining if timeout.pool is None else min(timeout.pool, remaining), + ), + ) return replace(document.call, timeout=remaining) @@ -264,6 +277,7 @@ def _prepare_document( provider: str, api_key: str | None, api_base: str | None, + transport_timeout: float | httpx.Timeout, kwargs: Mapping[str, object], ) -> _DocumentCall: text: Final = request.document_text(index) @@ -277,7 +291,7 @@ def _prepare_document( safety_identifier=request.safety_identifier, api_key=api_key, api_base=api_base, - timeout=request.timeout, + timeout=transport_timeout, custom_llm_provider=provider, extra_headers=request.extra_headers, kwargs={key: value for key, value in kwargs.items() if key not in _CHILD_OMIT}, @@ -294,6 +308,7 @@ def rerank_with_decisions( logging_obj: Logging, is_async: bool, strategy: Literal["pointwise", "multiple_choice"], + transport_timeout: float | httpx.Timeout, params: Mapping[str, object], kwargs: Mapping[str, object], ) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]: @@ -315,6 +330,7 @@ def rerank_with_decisions( provider=provider, api_key=api_key, api_base=api_base, + transport_timeout=transport_timeout, kwargs=kwargs, ) first: Final = prepare(0) diff --git a/litellm/rerank_api/main.py b/litellm/rerank_api/main.py index 86929d61434..95acb13784d 100644 --- a/litellm/rerank_api/main.py +++ b/litellm/rerank_api/main.py @@ -36,11 +36,10 @@ base_llm_http_handler = BaseLLMHTTPHandler() ################################################# -def _decision_rerank_timeout(timeout: object) -> float: - limit: Final = CompletionTimeout.as_limit(litellm.request_timeout if timeout is None else timeout) - if isinstance(limit, httpx.Timeout): - return float(litellm.request_timeout) if limit.read is None else limit.read - return limit +def _decision_rerank_timeout(timeout: float | httpx.Timeout) -> float: + if isinstance(timeout, httpx.Timeout): + return float(litellm.request_timeout) if timeout.read is None else timeout.read + return timeout @client @@ -173,6 +172,9 @@ def rerank( decision_strategy: Final = get_decision_rerank_strategy(model, litellm.LlmProviders(_custom_llm_provider)) if decision_strategy is not None: decision_kwargs: Final = _DECISION_KWARGS.validate_python(kwargs) + decision_timeout: Final = CompletionTimeout.as_limit( + litellm.request_timeout if optional_params.timeout is None else optional_params.timeout + ) from litellm.rerank_api.litellm_decisions_transformation.handler import ( rerank_with_decisions, ) @@ -185,6 +187,7 @@ def rerank( logging_obj=litellm_logging_obj, is_async=_is_async, strategy=decision_strategy, + transport_timeout=decision_timeout, params=_DECISION_KWARGS.validate_python( { **decision_kwargs, @@ -195,7 +198,7 @@ def rerank( "rank_fields": rank_fields, "max_chunks_per_doc": max_chunks_per_doc, "max_tokens_per_doc": max_tokens_per_doc, - "timeout": _decision_rerank_timeout(optional_params.timeout), + "timeout": _decision_rerank_timeout(decision_timeout), "extra_headers": decision_kwargs.get("extra_headers") or headers, } ), diff --git a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py index 232039defad..658e21940a0 100644 --- a/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py +++ b/tests/unit/rerank_api/litellm_decisions_transformation/test_handler.py @@ -1359,24 +1359,43 @@ async def test_proxy_catalog_model_keeps_native_rerank_transport(sync: bool, res @pytest.mark.parametrize("sync", [True, False]) @pytest.mark.parametrize( - "timeout", - [httpx.Timeout(12.0), httpx.Timeout(12.0, connect=2.0), httpx.Timeout(12.0, read=None)], + "timeout,expected_limits", + [ + (12.0, ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))), + (httpx.Timeout(12.0), ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))), + ( + httpx.Timeout(connect=2.0, read=12.0, write=9.0, pool=3.0), + ((2.0, 12.0, 9.0, 3.0), (2.0, 5.0, 5.0, 3.0)), + ), + (httpx.Timeout(12.0, read=None), ((12.0, 20.0, 12.0, 12.0), (12.0, 13.0, 12.0, 12.0))), + (httpx.Timeout(None), ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))), + (None, ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))), + ], ) -async def test_decision_rerank_accepts_httpx_timeout( - sync: bool, timeout: httpx.Timeout, respx_mock: respx.MockRouter +async def test_decision_rerank_preserves_phase_timeouts_within_remaining_deadline( + sync: bool, + timeout: float | httpx.Timeout | None, + expected_limits: tuple[tuple[float, float, float, float], tuple[float, float, float, float]], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ) -> None: + import litellm.rerank_api.litellm_decisions_transformation.handler as handler + + monkeypatch.setattr(litellm, "request_timeout", 20.0) route: Final = respx_mock.post(URL).mock(return_value=response()) - params: Final = { - "model": MODEL, - "query": "warranty", - "documents": ["one"], - "return_documents": False, - "api_key": "test-key", - "timeout": timeout, - } - result: Final = litellm.rerank(**params) if sync else await litellm.arerank(**params) - assert result.results == [{"index": 0, "relevance_score": 0.8}] - assert route.call_count == 1 + original_timeout: Final = httpx.Timeout(timeout).as_dict() + monkeypatch.setattr( + handler, + "time", + SimpleNamespace(monotonic=lambda: 100.0 + 7.0 * min(route.call_count, 1)), + ) + result: Final = await rerank(sync, documents=["one", "two"], return_documents=False, timeout=timeout, concurrency=1) + assert result.results == [{"index": 0, "relevance_score": 0.8}, {"index": 1, "relevance_score": 0.8}] + assert tuple(call.request.extensions["timeout"] for call in route.calls) == tuple( + {"connect": connect, "read": read, "write": write, "pool": pool} + for connect, read, write, pool in expected_limits + ) + assert httpx.Timeout(timeout).as_dict() == original_timeout @pytest.mark.parametrize("sync", [True, False])