fix(proxy): serialize reservation settlement and billing per request
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
yassin 2026-09-26 11:42:03 +00:00
parent 6aff9dbd9b
commit 511e689183
3 changed files with 76 additions and 62 deletions

View file

@ -629,27 +629,27 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
actual_tokens: int,
parent_otel_span: "Span | None",
) -> bool:
if stash.dynamic_reserved_tokens <= 0 or stash.dynamic_reservation_settled:
return False
stash.dynamic_reservation_settled = True
stash.dynamic_reservation_settled_tokens = actual_tokens
try:
await self.v3_limiter.async_increment_reservation_aware_tokens(
pipeline_operations=self.v3_limiter.build_project_reservation_ops(
targets=sorted(stash.dynamic_token_scopes),
reserved_scopes=stash.dynamic_token_scopes,
actual_tokens=actual_tokens,
reserved_tokens=stash.dynamic_reserved_tokens,
reservation_window_identities=stash.dynamic_reservation_windows,
),
parent_otel_span=parent_otel_span,
)
except Exception:
if stash.dynamic_reservation_settled_tokens == actual_tokens:
async with stash.dynamic_settlement_lock:
if stash.dynamic_reserved_tokens <= 0 or stash.dynamic_reservation_settled:
return False
stash.dynamic_reservation_settled = True
stash.dynamic_reservation_settled_tokens = actual_tokens
try:
await self.v3_limiter.async_increment_reservation_aware_tokens(
pipeline_operations=self.v3_limiter.build_project_reservation_ops(
targets=sorted(stash.dynamic_token_scopes),
reserved_scopes=stash.dynamic_token_scopes,
actual_tokens=actual_tokens,
reserved_tokens=stash.dynamic_reserved_tokens,
reservation_window_identities=stash.dynamic_reservation_windows,
),
parent_otel_span=parent_otel_span,
)
except Exception:
stash.dynamic_reservation_settled = False
stash.dynamic_reservation_settled_tokens = 0
raise
return True
raise
return True
async def _bill_settled_reservation_usage(
self,
@ -657,26 +657,26 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
actual_tokens: int,
parent_otel_span: "Span | None",
) -> None:
increment: Final = actual_tokens - stash.dynamic_reservation_settled_tokens
if increment <= 0:
return
previous: Final = stash.dynamic_reservation_settled_tokens
stash.dynamic_reservation_settled_tokens = actual_tokens
try:
await self.v3_limiter.async_increment_reservation_aware_tokens(
pipeline_operations=self.v3_limiter.build_project_reservation_ops(
targets=sorted(stash.dynamic_token_scopes),
reserved_scopes=frozenset(),
actual_tokens=increment,
reserved_tokens=0,
reservation_window_identities=stash.dynamic_reservation_windows,
),
parent_otel_span=parent_otel_span,
)
except Exception:
if stash.dynamic_reservation_settled_tokens == actual_tokens:
async with stash.dynamic_settlement_lock:
increment: Final = actual_tokens - stash.dynamic_reservation_settled_tokens
if increment <= 0:
return
previous: Final = stash.dynamic_reservation_settled_tokens
stash.dynamic_reservation_settled_tokens = actual_tokens
try:
await self.v3_limiter.async_increment_reservation_aware_tokens(
pipeline_operations=self.v3_limiter.build_project_reservation_ops(
targets=sorted(stash.dynamic_token_scopes),
reserved_scopes=frozenset(),
actual_tokens=increment,
reserved_tokens=0,
reservation_window_identities=stash.dynamic_reservation_windows,
),
parent_otel_span=parent_otel_span,
)
except Exception:
stash.dynamic_reservation_settled_tokens = previous
raise
raise
def _queue_weights(self, model_group_info: ModelGroupInfo, fairness: FairnessSettings) -> Mapping[str, float]:
normalized: Final = self._normalize_priority_weights(model_group_info)

View file

@ -592,6 +592,7 @@ class RequestRateLimiterStash:
)
dynamic_reservation_settled: bool = False
dynamic_reservation_settled_tokens: int = 0
dynamic_settlement_lock: asyncio.Lock = field(default_factory=asyncio.Lock)
dynamic_admission_bypassed: bool = False
fairness_queue_wait_seconds: float = 0.0

View file

@ -2755,7 +2755,7 @@ async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_to
monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (6, 0, 0))
data = {"model": model, "litellm_call_id": "racing-settle"}
async def run_race() -> None:
async def run_race() -> tuple[bool, int]:
stash = get_or_create_request_stash()
stash.dynamic_reserved_tokens = 40
stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)})
@ -2767,28 +2767,35 @@ async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_to
user_api_key_dict=_prod_user(),
)
)
await asyncio.wait_for(first_call_seen.wait(), timeout=5)
await handler.async_log_success_event(
kwargs=_success_kwargs(model, "racing-settle", "prod"),
response_obj=ModelResponse(
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
),
start_time=None,
end_time=None,
success_task = asyncio.create_task(
handler.async_log_success_event(
kwargs=_success_kwargs(model, "racing-settle", "prod"),
response_obj=ModelResponse(
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
),
start_time=None,
end_time=None,
)
)
await asyncio.wait_for(first_call_seen.wait(), timeout=5)
for _ in range(3):
await asyncio.sleep(0)
settle_gate.set()
await failure_task
await asyncio.gather(failure_task, success_task)
return stash.dynamic_reservation_settled, stash.dynamic_reservation_settled_tokens
await asyncio.create_task(run_race())
settled, settled_tokens = await asyncio.create_task(run_race())
assert len(calls) == 2
first_ops = calls[0][1]["pipeline_operations"]
assert all(op["increment_value"] == -34 for op in first_ops)
second_ops = calls[1][1]["pipeline_operations"]
assert len(second_ops) == 1
for op in second_ops:
assert op["increment_value"] == 4
assert all(op["increment_value"] == 4 for op in second_ops)
assert settled
assert settled_tokens == 10
@pytest.mark.asyncio
async def test_failed_settlement_write_keeps_the_record_of_a_concurrent_billing(monkeypatch):
async def test_failed_settlement_write_lets_the_waiting_success_settle_at_actual_usage(monkeypatch):
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
from litellm.types.utils import ModelResponse, Usage
@ -2828,20 +2835,26 @@ async def test_failed_settlement_write_keeps_the_record_of_a_concurrent_billing(
user_api_key_dict=_prod_user(),
)
)
await asyncio.wait_for(first_call_seen.wait(), timeout=5)
await handler.async_log_success_event(
kwargs=_success_kwargs(model, "failed-settle-race", "prod"),
response_obj=ModelResponse(
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
),
start_time=None,
end_time=None,
success_task = asyncio.create_task(
handler.async_log_success_event(
kwargs=_success_kwargs(model, "failed-settle-race", "prod"),
response_obj=ModelResponse(
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
),
start_time=None,
end_time=None,
)
)
await asyncio.wait_for(first_call_seen.wait(), timeout=5)
for _ in range(3):
await asyncio.sleep(0)
settle_gate.set()
await failure_task
await asyncio.gather(failure_task, success_task)
return stash.dynamic_reservation_settled, stash.dynamic_reservation_settled_tokens
settled, settled_tokens = await asyncio.create_task(run_race())
assert len(calls) == 2
second_ops = calls[1][1]["pipeline_operations"]
assert all(op["increment_value"] == -30 for op in second_ops)
assert settled
assert settled_tokens == 10