mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(rerank): preserve HTTP phase timeout limits
Keep connection, read, write, and pool limits within the remaining decision-rerank deadline.
This commit is contained in:
parent
314c52bcf4
commit
dbfb175303
3 changed files with 60 additions and 22 deletions
|
|
@ -10,6 +10,8 @@ from queue import Empty, SimpleQueue
|
|||
from threading import Event
|
||||
from typing import Final, Literal
|
||||
|
||||
import httpx
|
||||
|
||||
import litellm
|
||||
from litellm.decisions.call import (
|
||||
DecisionsCall,
|
||||
|
|
@ -53,6 +55,17 @@ def _call_with_deadline(document: _DocumentCall, deadline: float) -> DecisionsCa
|
|||
remaining: Final = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return _timeout(document.call.requested_model, document.call.custom_llm_provider)
|
||||
timeout: Final = document.call.timeout
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
return replace(
|
||||
document.call,
|
||||
timeout=httpx.Timeout(
|
||||
connect=remaining if timeout.connect is None else min(timeout.connect, remaining),
|
||||
read=remaining if timeout.read is None else min(timeout.read, remaining),
|
||||
write=remaining if timeout.write is None else min(timeout.write, remaining),
|
||||
pool=remaining if timeout.pool is None else min(timeout.pool, remaining),
|
||||
),
|
||||
)
|
||||
return replace(document.call, timeout=remaining)
|
||||
|
||||
|
||||
|
|
@ -264,6 +277,7 @@ def _prepare_document(
|
|||
provider: str,
|
||||
api_key: str | None,
|
||||
api_base: str | None,
|
||||
transport_timeout: float | httpx.Timeout,
|
||||
kwargs: Mapping[str, object],
|
||||
) -> _DocumentCall:
|
||||
text: Final = request.document_text(index)
|
||||
|
|
@ -277,7 +291,7 @@ def _prepare_document(
|
|||
safety_identifier=request.safety_identifier,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
timeout=request.timeout,
|
||||
timeout=transport_timeout,
|
||||
custom_llm_provider=provider,
|
||||
extra_headers=request.extra_headers,
|
||||
kwargs={key: value for key, value in kwargs.items() if key not in _CHILD_OMIT},
|
||||
|
|
@ -294,6 +308,7 @@ def rerank_with_decisions(
|
|||
logging_obj: Logging,
|
||||
is_async: bool,
|
||||
strategy: Literal["pointwise", "multiple_choice"],
|
||||
transport_timeout: float | httpx.Timeout,
|
||||
params: Mapping[str, object],
|
||||
kwargs: Mapping[str, object],
|
||||
) -> DecisionRerankResponse | Awaitable[DecisionRerankResponse]:
|
||||
|
|
@ -315,6 +330,7 @@ def rerank_with_decisions(
|
|||
provider=provider,
|
||||
api_key=api_key,
|
||||
api_base=api_base,
|
||||
transport_timeout=transport_timeout,
|
||||
kwargs=kwargs,
|
||||
)
|
||||
first: Final = prepare(0)
|
||||
|
|
|
|||
|
|
@ -36,11 +36,10 @@ base_llm_http_handler = BaseLLMHTTPHandler()
|
|||
#################################################
|
||||
|
||||
|
||||
def _decision_rerank_timeout(timeout: object) -> float:
|
||||
limit: Final = CompletionTimeout.as_limit(litellm.request_timeout if timeout is None else timeout)
|
||||
if isinstance(limit, httpx.Timeout):
|
||||
return float(litellm.request_timeout) if limit.read is None else limit.read
|
||||
return limit
|
||||
def _decision_rerank_timeout(timeout: float | httpx.Timeout) -> float:
|
||||
if isinstance(timeout, httpx.Timeout):
|
||||
return float(litellm.request_timeout) if timeout.read is None else timeout.read
|
||||
return timeout
|
||||
|
||||
|
||||
@client
|
||||
|
|
@ -173,6 +172,9 @@ def rerank(
|
|||
decision_strategy: Final = get_decision_rerank_strategy(model, litellm.LlmProviders(_custom_llm_provider))
|
||||
if decision_strategy is not None:
|
||||
decision_kwargs: Final = _DECISION_KWARGS.validate_python(kwargs)
|
||||
decision_timeout: Final = CompletionTimeout.as_limit(
|
||||
litellm.request_timeout if optional_params.timeout is None else optional_params.timeout
|
||||
)
|
||||
from litellm.rerank_api.litellm_decisions_transformation.handler import (
|
||||
rerank_with_decisions,
|
||||
)
|
||||
|
|
@ -185,6 +187,7 @@ def rerank(
|
|||
logging_obj=litellm_logging_obj,
|
||||
is_async=_is_async,
|
||||
strategy=decision_strategy,
|
||||
transport_timeout=decision_timeout,
|
||||
params=_DECISION_KWARGS.validate_python(
|
||||
{
|
||||
**decision_kwargs,
|
||||
|
|
@ -195,7 +198,7 @@ def rerank(
|
|||
"rank_fields": rank_fields,
|
||||
"max_chunks_per_doc": max_chunks_per_doc,
|
||||
"max_tokens_per_doc": max_tokens_per_doc,
|
||||
"timeout": _decision_rerank_timeout(optional_params.timeout),
|
||||
"timeout": _decision_rerank_timeout(decision_timeout),
|
||||
"extra_headers": decision_kwargs.get("extra_headers") or headers,
|
||||
}
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1359,24 +1359,43 @@ async def test_proxy_catalog_model_keeps_native_rerank_transport(sync: bool, res
|
|||
|
||||
@pytest.mark.parametrize("sync", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"timeout",
|
||||
[httpx.Timeout(12.0), httpx.Timeout(12.0, connect=2.0), httpx.Timeout(12.0, read=None)],
|
||||
"timeout,expected_limits",
|
||||
[
|
||||
(12.0, ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))),
|
||||
(httpx.Timeout(12.0), ((12.0, 12.0, 12.0, 12.0), (5.0, 5.0, 5.0, 5.0))),
|
||||
(
|
||||
httpx.Timeout(connect=2.0, read=12.0, write=9.0, pool=3.0),
|
||||
((2.0, 12.0, 9.0, 3.0), (2.0, 5.0, 5.0, 3.0)),
|
||||
),
|
||||
(httpx.Timeout(12.0, read=None), ((12.0, 20.0, 12.0, 12.0), (12.0, 13.0, 12.0, 12.0))),
|
||||
(httpx.Timeout(None), ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))),
|
||||
(None, ((20.0, 20.0, 20.0, 20.0), (13.0, 13.0, 13.0, 13.0))),
|
||||
],
|
||||
)
|
||||
async def test_decision_rerank_accepts_httpx_timeout(
|
||||
sync: bool, timeout: httpx.Timeout, respx_mock: respx.MockRouter
|
||||
async def test_decision_rerank_preserves_phase_timeouts_within_remaining_deadline(
|
||||
sync: bool,
|
||||
timeout: float | httpx.Timeout | None,
|
||||
expected_limits: tuple[tuple[float, float, float, float], tuple[float, float, float, float]],
|
||||
respx_mock: respx.MockRouter,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import litellm.rerank_api.litellm_decisions_transformation.handler as handler
|
||||
|
||||
monkeypatch.setattr(litellm, "request_timeout", 20.0)
|
||||
route: Final = respx_mock.post(URL).mock(return_value=response())
|
||||
params: Final = {
|
||||
"model": MODEL,
|
||||
"query": "warranty",
|
||||
"documents": ["one"],
|
||||
"return_documents": False,
|
||||
"api_key": "test-key",
|
||||
"timeout": timeout,
|
||||
}
|
||||
result: Final = litellm.rerank(**params) if sync else await litellm.arerank(**params)
|
||||
assert result.results == [{"index": 0, "relevance_score": 0.8}]
|
||||
assert route.call_count == 1
|
||||
original_timeout: Final = httpx.Timeout(timeout).as_dict()
|
||||
monkeypatch.setattr(
|
||||
handler,
|
||||
"time",
|
||||
SimpleNamespace(monotonic=lambda: 100.0 + 7.0 * min(route.call_count, 1)),
|
||||
)
|
||||
result: Final = await rerank(sync, documents=["one", "two"], return_documents=False, timeout=timeout, concurrency=1)
|
||||
assert result.results == [{"index": 0, "relevance_score": 0.8}, {"index": 1, "relevance_score": 0.8}]
|
||||
assert tuple(call.request.extensions["timeout"] for call in route.calls) == tuple(
|
||||
{"connect": connect, "read": read, "write": write, "pool": pool}
|
||||
for connect, read, write, pool in expected_limits
|
||||
)
|
||||
assert httpx.Timeout(timeout).as_dict() == original_timeout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sync", [True, False])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue