fix(rerank): preserve HTTP phase timeout limits

Keep connection, read, write, and pool limits within the remaining decision-rerank deadline.
This commit is contained in:
ishaan-j-berri 2026-10-24 10:13:18 -07:00
parent 314c52bcf4
commit dbfb175303
3 changed files with 60 additions and 22 deletions

View file

@ -10,6 +10,8 @@ from queue import Empty, SimpleQueue
from threading import Event
from typing import Final, Literal
import httpx
import litellm
from litellm.decisions.call import (
DecisionsCall,
@ -53,6 +55,17 @@ def _call_with_deadline(document: _DocumentCall, deadline: float) -> DecisionsCa
remaining: Final = deadline - time.monotonic()
if remaining <= 0:
return _timeout(document.call.requested_model, document.call.custom_llm_provider)
timeout: Final = document.call.timeout
if isinstance(timeout, httpx.Timeout):
return replace(
document.call,
timeout=httpx.Timeout(
connect=remaining if timeout.connect is None else min(timeout.connect, remaining),
read=remaining if timeout.read is None else min(timeout.read, remaining),
write=remaining if timeout.write is None else min(timeout.write, remaining),
pool=remaining if timeout.pool is None else min(timeout.pool, remaining),
),
)
return replace(document.call, timeout=remaining)
@ -264,6 +277,7 @@ def _prepare_document(
provider: str,
api_key: str | None,
api_base: str | None,
transport_timeout: float | httpx.Timeout,
kwargs: Mapping[str, object],
) -> _DocumentCall:
text: Final = request.document_text(index)
@ -277,7 +291,7 @@ def _prepare_document(
safety_identifier=request.safety_identifier,
api_key=api_key,
api_base=api_base,
timeout=request.timeout,
timeout=transport_timeout,
custom_llm_provider=provider,
extra_headers=request.extra_headers,
kwargs={key: value for key, value in kwargs.items() if key not in _CHILD_OMIT},
@ -294,6 +308,7 @@ def rerank_with_decisions(
logging_obj: Logging,
is_async: bool,
strategy: Literal["pointwise", "multiple_choice"],
transport_timeout: float | httpx.Timeout,
params: Mapping[str, object],
kwargs: Mapping[str, object],
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
@ -315,6 +330,7 @@ def rerank_with_decisions(
provider=provider,
api_key=api_key,
api_base=api_base,
transport_timeout=transport_timeout,
kwargs=kwargs,
)
first: Final = prepare(0)

View file

@ -36,11 +36,10 @@ base_llm_http_handler = BaseLLMHTTPHandler()
#################################################
def _decision_rerank_timeout(timeout: object) -> float:
limit: Final = CompletionTimeout.as_limit(litellm.request_timeout if timeout is None else timeout)
if isinstance(limit, httpx.Timeout):
return float(litellm.request_timeout) if limit.read is None else limit.read
return limit
def _decision_rerank_timeout(timeout: float | httpx.Timeout) -> float:
if isinstance(timeout, httpx.Timeout):
return float(litellm.request_timeout) if timeout.read is None else timeout.read
return timeout
@client
@ -173,6 +172,9 @@ def rerank(
decision_strategy: Final = get_decision_rerank_strategy(model, litellm.LlmProviders(_custom_llm_provider))
if decision_strategy is not None:
decision_kwargs: Final = _DECISION_KWARGS.validate_python(kwargs)
decision_timeout: Final = CompletionTimeout.as_limit(
litellm.request_timeout if optional_params.timeout is None else optional_params.timeout
)
from litellm.rerank_api.litellm_decisions_transformation.handler import (
rerank_with_decisions,
)
@ -185,6 +187,7 @@ def rerank(
logging_obj=litellm_logging_obj,
is_async=_is_async,
strategy=decision_strategy,
transport_timeout=decision_timeout,
params=_DECISION_KWARGS.validate_python(
{
**decision_kwargs,
@ -195,7 +198,7 @@ def rerank(
"rank_fields": rank_fields,
"max_chunks_per_doc": max_chunks_per_doc,
"max_tokens_per_doc": max_tokens_per_doc,
"timeout": _decision_rerank_timeout(optional_params.timeout),
"timeout": _decision_rerank_timeout(decision_timeout),
"extra_headers": decision_kwargs.get("extra_headers") or headers,
}
),

View file

@ -1359,24 +1359,43 @@ async def test_proxy_catalog_model_keeps_native_rerank_transport(sync: bool, res
@pytest.mark.parametrize("sync", [True, False])
@pytest.mark.parametrize(
"timeout",
[httpx.Timeout(12.0), httpx.Timeout(12.0, connect=2.0), httpx.Timeout(12.0, read=None)],
"timeout,expected_limits",
[
(12.0, ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))),
(httpx.Timeout(12.0), ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))),
(
httpx.Timeout(connect=2.0, read=12.0, write=9.0, pool=3.0),
((2.0, 12.0, 9.0, 3.0), (2.0, 5.0, 5.0, 3.0)),
),
(httpx.Timeout(12.0, read=None), ((12.0, 20.0, 12.0, 12.0), (12.0, 13.0, 12.0, 12.0))),
(httpx.Timeout(None), ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))),
(None, ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))),
],
)
async def test_decision_rerank_accepts_httpx_timeout(
sync: bool, timeout: httpx.Timeout, respx_mock: respx.MockRouter
async def test_decision_rerank_preserves_phase_timeouts_within_remaining_deadline(
sync: bool,
timeout: float | httpx.Timeout | None,
expected_limits: tuple[tuple[float, float, float, float], tuple[float, float, float, float]],
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
) -> None:
import litellm.rerank_api.litellm_decisions_transformation.handler as handler
monkeypatch.setattr(litellm, "request_timeout", 20.0)
route: Final = respx_mock.post(URL).mock(return_value=response())
params: Final = {
"model": MODEL,
"query": "warranty",
"documents": ["one"],
"return_documents": False,
"api_key": "test-key",
"timeout": timeout,
}
result: Final = litellm.rerank(**params) if sync else await litellm.arerank(**params)
assert result.results == [{"index": 0, "relevance_score": 0.8}]
assert route.call_count == 1
original_timeout: Final = httpx.Timeout(timeout).as_dict()
monkeypatch.setattr(
handler,
"time",
SimpleNamespace(monotonic=lambda: 100.0 + 7.0 * min(route.call_count, 1)),
)
result: Final = await rerank(sync, documents=["one", "two"], return_documents=False, timeout=timeout, concurrency=1)
assert result.results == [{"index": 0, "relevance_score": 0.8}, {"index": 1, "relevance_score": 0.8}]
assert tuple(call.request.extensions["timeout"] for call in route.calls) == tuple(
{"connect": connect, "read": read, "write": write, "pool": pool}
for connect, read, write, pool in expected_limits
)
assert httpx.Timeout(timeout).as_dict() == original_timeout
@pytest.mark.parametrize("sync", [True, False])