diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 9afc6d90f68..4c1f1246fe6 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -632,6 +632,7 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): if stash.dynamic_reserved_tokens <= 0 or stash.dynamic_reservation_settled: return False stash.dynamic_reservation_settled = True + stash.dynamic_reservation_settled_tokens = actual_tokens try: await self.v3_limiter.async_increment_reservation_aware_tokens( pipeline_operations=self.v3_limiter.build_project_reservation_ops( @@ -643,9 +644,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): ), parent_otel_span=parent_otel_span, ) - stash.dynamic_reservation_settled_tokens = actual_tokens except Exception: stash.dynamic_reservation_settled = False + stash.dynamic_reservation_settled_tokens = 0 raise return True @@ -658,17 +659,22 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): increment: Final = actual_tokens - stash.dynamic_reservation_settled_tokens if increment <= 0: return - await self.v3_limiter.async_increment_reservation_aware_tokens( - pipeline_operations=self.v3_limiter.build_project_reservation_ops( - targets=sorted(stash.dynamic_token_scopes), - reserved_scopes=frozenset(), - actual_tokens=increment, - reserved_tokens=0, - reservation_window_identities=stash.dynamic_reservation_windows, - ), - parent_otel_span=parent_otel_span, - ) + previous: Final = stash.dynamic_reservation_settled_tokens stash.dynamic_reservation_settled_tokens = actual_tokens + try: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=self.v3_limiter.build_project_reservation_ops( + targets=sorted(stash.dynamic_token_scopes), + reserved_scopes=frozenset(), + actual_tokens=increment, + reserved_tokens=0, + reservation_window_identities=stash.dynamic_reservation_windows, + ), + parent_otel_span=parent_otel_span, + ) + except Exception: + stash.dynamic_reservation_settled_tokens = previous + raise def _queue_weights(self, model_group_info: ModelGroupInfo, fairness: FairnessSettings) -> Mapping[str, float]: normalized: Final = self._normalize_priority_weights(model_group_info) diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 66307c4478f..14cb7405111 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2725,3 +2725,63 @@ async def test_post_call_failure_hook_records_recovered_usage_as_settled_tokens( settled_tokens = await asyncio.create_task(reject_with_recovered_usage()) assert settled_tokens == 6 settle_mock.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_tokens(monkeypatch): + from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash + from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass + from litellm.types.utils import ModelResponse, Usage + + model = "fairness-racing-settle-model" + _enable_fairness( + monkeypatch, + FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)), + ) + dual_cache = DualCache() + handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache) + handler.update_variables(llm_router=_fairness_router(model, tpm=100_000)) + settle_gate = asyncio.Event() + first_call_seen = asyncio.Event() + calls: list[tuple] = [] + + async def gated_increment(*args, **kwargs): + calls.append((args, kwargs)) + if len(calls) == 1: + first_call_seen.set() + await settle_gate.wait() + + monkeypatch.setattr(handler.v3_limiter, "async_increment_reservation_aware_tokens", gated_increment) + monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (6, 0, 0)) + data = {"model": model, "litellm_call_id": "racing-settle"} + + async def run_race() -> None: + stash = get_or_create_request_stash() + stash.dynamic_reserved_tokens = 40 + stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)}) + + failure_task = asyncio.create_task( + handler.async_post_call_failure_hook( + request_data=data, + original_exception=Exception("post-call guardrail rejected"), + user_api_key_dict=_prod_user(), + ) + ) + await asyncio.wait_for(first_call_seen.wait(), timeout=5) + await handler.async_log_success_event( + kwargs=_success_kwargs(model, "racing-settle", "prod"), + response_obj=ModelResponse( + model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) + ), + start_time=None, + end_time=None, + ) + settle_gate.set() + await failure_task + + await asyncio.create_task(run_race()) + assert len(calls) == 2 + second_ops = calls[1][1]["pipeline_operations"] + assert len(second_ops) == 1 + for op in second_ops: + assert op["increment_value"] == 4