fix(rerank): reconcile late usage before failure logging

This commit is contained in:
Ishaan 2026-10-23 16:48:06 -07:00
parent 3a263b27b3
commit 72bcd5d945
2 changed files with 253 additions and 70 deletions

View file

@ -213,17 +213,18 @@ def _rerank(
executor: Final = ThreadPoolExecutor(max_workers=workers)
futures: Final = tuple(executor.submit(run, range(worker, count, workers)) for worker in range(workers))
try:
wait(futures, timeout=max(0.0, deadline - time.monotonic()))
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
while True:
try:
outcomes.append(completed.get_nowait())
except Empty:
break
return _finish(outcomes, request, logging_obj, len(outcomes) < count, model, provider)
waited: Final = wait(futures, timeout=max(0.0, deadline - time.monotonic()))
timed_out: Final = bool(waited.not_done) or time.monotonic() >= deadline
finally:
stopped.set()
executor.shutdown(wait=False, cancel_futures=True)
executor.shutdown(wait=True, cancel_futures=True)
outcomes: Final[list[DecisionRerankResult]] = [] # mutable-ok: retain completed results for partial billing
while True:
try:
outcomes.append(completed.get_nowait())
except Empty:
break
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
async def _arerank(
@ -262,21 +263,23 @@ async def _arerank(
tasks: Final = tuple(asyncio.create_task(run(range(worker, count, workers))) for worker in range(workers))
try:
await asyncio.wait(tasks, timeout=max(0.0, deadline - time.monotonic()))
return _finish(outcomes, request, logging_obj, len(outcomes) < count, model, provider)
try:
_, pending = await asyncio.wait(tasks, timeout=max(0.0, deadline - time.monotonic()))
timed_out: Final = bool(pending) or time.monotonic() >= deadline
finally:
stopped.set()
cancellation_requested.set()
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
except asyncio.CancelledError:
response_on_cancel, _ = LiteLLMDecisionsRerankConfig.transform_decisions_response(
outcomes, request, model, provider
)
_record_partial(response_on_cancel, logging_obj)
raise
finally:
stopped.set()
cancellation_requested.set()
for task in tasks:
if not task.done():
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
return _finish(outcomes, request, logging_obj, timed_out or len(outcomes) < count, model, provider)
def _prepare_document(

View file

@ -1,6 +1,7 @@
import asyncio
import json
from threading import Event
from types import SimpleNamespace
from typing import Final
from unittest.mock import patch
@ -655,39 +656,212 @@ async def test_custom_transport_configuration_and_untrusted_text(strategy, respx
assert route.call_count == (1 if strategy == "multiple_choice" else 2)
@pytest.mark.parametrize("sync", [True, False])
async def test_deadline_bounds_fanout_and_never_starts_queued_requests(sync, respx_mock):
import threading
import time
@pytest.fixture
def deadline_expired(monkeypatch):
from litellm.rerank_api.litellm_decisions_transformation import handler
release = threading.Event()
started = []
expired = Event()
monkeypatch.setattr(
handler,
"time",
SimpleNamespace(monotonic=lambda: 102.0 if expired.is_set() else 100.0),
)
return expired
def sync_score(request):
started.append(request)
release.wait(5)
return response()
async def async_score(request):
started.append(request)
await asyncio.Event().wait()
@pytest.mark.parametrize("refused", [False, True])
@pytest.mark.parametrize("document_count", [1, 3])
@pytest.mark.parametrize("settled_before_wait_returns", [False, True])
async def test_sync_deadline_bills_late_responses_once_without_dispatching_queued_documents(
refused,
document_count,
settled_before_wait_returns,
respx_mock,
monkeypatch,
deadline_expired,
):
from concurrent.futures import ThreadPoolExecutor, wait
respx_mock.post(URL).mock(side_effect=sync_score if sync else async_score)
start = time.monotonic()
try:
with pytest.raises(litellm.Timeout):
await rerank(
sync,
documents=["one", "two", "three"],
timeout=0.05,
concurrency=1,
from litellm.rerank_api.litellm_decisions_transformation import handler
logger = RecordingLogger()
started = Event()
release = Event()
logged = Event()
def record(kwargs, response_obj, start_time, end_time):
logger.events.append(kwargs["standard_logging_object"])
logged.set()
def score(request):
started.set()
assert release.wait(5)
return (
httpx.Response(
200,
json={"answers": {}, "usage": {"input_tokens": 10, "output_tokens": 3}},
)
assert time.monotonic() - start < 0.75
assert len(started) == 1
finally:
release.set()
await asyncio.sleep(0.02)
assert len(started) == 1
if refused
else response(output_tokens=3)
)
def wait_until_deadline(futures, timeout):
assert started.wait(5)
deadline_expired.set()
if settled_before_wait_returns:
release.set()
return wait(futures)
return wait(futures, timeout=0)
class ReleasingExecutor(ThreadPoolExecutor):
def shutdown(self, wait=True, *, cancel_futures=False):
release.set()
return super().shutdown(wait=wait, cancel_futures=cancel_futures)
monkeypatch.setattr(logger, "log_failure_event", record)
monkeypatch.setattr(litellm, "callbacks", [logger])
monkeypatch.setattr(handler, "wait", wait_until_deadline)
monkeypatch.setattr(handler, "ThreadPoolExecutor", ReleasingExecutor)
route = respx_mock.post(URL).mock(side_effect=score)
error = litellm.ContentPolicyViolationError if refused else litellm.Timeout
with pytest.raises(error):
await rerank(True, documents=["document"] * document_count, timeout=1, concurrency=1)
assert await asyncio.to_thread(logged.wait, 5)
assert route.call_count == 1
assert len(logger.events) == 1
event = logger.events[0]
assert event["status"] == "failure"
assert (
event["prompt_tokens"],
event["completion_tokens"],
event["total_tokens"],
) == (10, 3, 13)
assert event["response_cost"] == pytest.approx(10 * RATE)
async def test_async_deadline_cancels_inflight_requests_without_dispatching_queued_documents(
respx_mock, monkeypatch, deadline_expired
):
from litellm.rerank_api.litellm_decisions_transformation import handler
started = asyncio.Event()
cancelled = asyncio.Event()
requests = []
wait = asyncio.wait
async def score(request):
requests.append(request)
started.set()
try:
await asyncio.Event().wait()
finally:
cancelled.set()
async def wait_until_deadline(tasks, timeout):
await asyncio.wait_for(started.wait(), 5)
deadline_expired.set()
return await wait(tasks, timeout=0)
monkeypatch.setattr(handler.asyncio, "wait", wait_until_deadline)
respx_mock.post(URL).mock(side_effect=score)
with pytest.raises(litellm.Timeout):
await rerank(False, documents=["one", "two", "three"], timeout=1, concurrency=1)
assert cancelled.is_set()
assert len(requests) == 1
@pytest.mark.parametrize("refused", [False, True])
@pytest.mark.parametrize("document_count", [1, 3])
async def test_async_deadline_bills_responses_returned_during_cancellation_once(
refused, document_count, respx_mock, monkeypatch, deadline_expired
):
from litellm.rerank_api.litellm_decisions_transformation import handler
logger = RecordingLogger()
started = asyncio.Event()
wait = asyncio.wait
async def score(request):
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
return (
httpx.Response(
200,
json={
"answers": {},
"usage": {"input_tokens": 10, "output_tokens": 3},
},
)
if refused
else response(output_tokens=3)
)
async def wait_until_deadline(tasks, timeout):
await asyncio.wait_for(started.wait(), 5)
deadline_expired.set()
return await wait(tasks, timeout=0)
monkeypatch.setattr(litellm, "callbacks", [logger])
monkeypatch.setattr(handler.asyncio, "wait", wait_until_deadline)
route = respx_mock.post(URL).mock(side_effect=score)
error = litellm.ContentPolicyViolationError if refused else litellm.Timeout
with pytest.raises(error):
await rerank(False, documents=["document"] * document_count, timeout=1, concurrency=1)
await flush_logging()
assert route.call_count == 1
assert len(logger.events) == 1
event = logger.events[0]
assert event["status"] == "failure"
assert (
event["prompt_tokens"],
event["completion_tokens"],
event["total_tokens"],
) == (10, 3, 13)
assert event["response_cost"] == pytest.approx(10 * RATE)
async def test_caller_cancellation_retains_usage_returned_during_transport_cleanup(
respx_mock,
):
from litellm.rerank_api.litellm_decisions_transformation.handler import (
DecisionRerankLoggingCapture,
)
started = asyncio.Event()
capture = DecisionRerankLoggingCapture("cancelled-rerank")
async def score(request):
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
return response(output_tokens=3)
route = respx_mock.post(URL).mock(side_effect=score)
with capture.capture():
task = asyncio.create_task(
rerank(
False,
documents=["one", "two"],
concurrency=1,
litellm_call_id=capture.call_id,
)
)
await asyncio.wait_for(started.wait(), 5)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert route.call_count == 1
assert capture.logging_obj is not None
usage = capture.logging_obj.model_call_details["combined_usage_object"]
assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (
10,
3,
13,
)
assert capture.logging_obj.model_call_details["response_cost"] == pytest.approx(10 * RATE)
async def test_direct_token_price_override_is_respected(respx_mock):
@ -950,15 +1124,18 @@ async def test_cancelled_child_fails_whole_request_and_bills_completed_usage(res
@pytest.mark.parametrize("sync", [True, False])
async def test_request_preparation_is_bounded_and_stops_at_deadline(sync, respx_mock, monkeypatch):
import threading
async def test_request_preparation_is_bounded_and_stops_at_deadline(sync, respx_mock, monkeypatch, deadline_expired):
from concurrent.futures import wait
from threading import Barrier
from litellm.rerank_api.litellm_decisions_transformation import handler
prepared = []
started = []
release = threading.Event()
sync_started = Barrier(2)
async_started = asyncio.Event()
original = handler._prepare_document
async_wait = asyncio.wait
def prepare(index, *args, **kwargs):
prepared.append(index)
@ -966,36 +1143,39 @@ async def test_request_preparation_is_bounded_and_stops_at_deadline(sync, respx_
def sync_score(request):
started.append(request)
release.wait(5)
sync_started.wait(timeout=5)
deadline_expired.set()
return response()
async def async_score(request):
started.append(request)
await asyncio.Event().wait()
if len(started) == 2:
deadline_expired.set()
async_started.set()
await asyncio.wait_for(async_started.wait(), 5)
return response()
async def wait_for_started_tasks(tasks, timeout):
return await async_wait(tasks)
monkeypatch.setattr(handler, "_prepare_document", prepare)
monkeypatch.setattr(handler, "wait", lambda futures, timeout: wait(futures))
monkeypatch.setattr(handler.asyncio, "wait", wait_for_started_tasks)
respx_mock.post(URL).mock(side_effect=sync_score if sync else async_score)
try:
with pytest.raises(litellm.Timeout):
await rerank(
sync,
query="long query " * 1000,
documents=[""] * 100,
timeout=0.05,
concurrency=2,
)
assert len(prepared) == 2
assert len(started) == 2
finally:
release.set()
await asyncio.sleep(0.02)
with pytest.raises(litellm.Timeout):
await rerank(
sync,
query="long query " * 1000,
documents=[""] * 100,
timeout=1,
concurrency=2,
)
assert len(prepared) == 2
assert len(started) == 2
@pytest.mark.parametrize("sync", [True, False])
async def test_preparation_time_counts_toward_request_deadline(sync, respx_mock, monkeypatch):
import time
async def test_preparation_time_counts_toward_request_deadline(sync, respx_mock, monkeypatch, deadline_expired):
from litellm.rerank_api.litellm_decisions_transformation import handler
original = handler._prepare_document
@ -1003,13 +1183,13 @@ async def test_preparation_time_counts_toward_request_deadline(sync, respx_mock,
def prepare(index, *args, **kwargs):
prepared.append(index)
time.sleep(0.03)
deadline_expired.set()
return original(index, *args, **kwargs)
monkeypatch.setattr(handler, "_prepare_document", prepare)
route = respx_mock.post(URL).mock(return_value=response())
with pytest.raises(litellm.Timeout):
await rerank(sync, documents=["one", "two"], timeout=0.01)
await rerank(sync, documents=["one", "two"], timeout=1)
assert prepared == [0]
assert route.call_count == 0