diff --git a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py index 53810d8a8cb..8cf4be68f73 100644 --- a/litellm/proxy/hooks/dynamic_rate_limiter_v3.py +++ b/litellm/proxy/hooks/dynamic_rate_limiter_v3.py @@ -629,27 +629,27 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): actual_tokens: int, parent_otel_span: "Span | None", ) -> bool: - if stash.dynamic_reserved_tokens <= 0 or stash.dynamic_reservation_settled: - return False - stash.dynamic_reservation_settled = True - stash.dynamic_reservation_settled_tokens = actual_tokens - try: - await self.v3_limiter.async_increment_reservation_aware_tokens( - pipeline_operations=self.v3_limiter.build_project_reservation_ops( - targets=sorted(stash.dynamic_token_scopes), - reserved_scopes=stash.dynamic_token_scopes, - actual_tokens=actual_tokens, - reserved_tokens=stash.dynamic_reserved_tokens, - reservation_window_identities=stash.dynamic_reservation_windows, - ), - parent_otel_span=parent_otel_span, - ) - except Exception: - if stash.dynamic_reservation_settled_tokens == actual_tokens: + async with stash.dynamic_settlement_lock: + if stash.dynamic_reserved_tokens <= 0 or stash.dynamic_reservation_settled: + return False + stash.dynamic_reservation_settled = True + stash.dynamic_reservation_settled_tokens = actual_tokens + try: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=self.v3_limiter.build_project_reservation_ops( + targets=sorted(stash.dynamic_token_scopes), + reserved_scopes=stash.dynamic_token_scopes, + actual_tokens=actual_tokens, + reserved_tokens=stash.dynamic_reserved_tokens, + reservation_window_identities=stash.dynamic_reservation_windows, + ), + parent_otel_span=parent_otel_span, + ) + except Exception: stash.dynamic_reservation_settled = False stash.dynamic_reservation_settled_tokens = 0 - raise - return True + raise + return True async def _bill_settled_reservation_usage( self, @@ -657,26 +657,26 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger): actual_tokens: int, parent_otel_span: "Span | None", ) -> None: - increment: Final = actual_tokens - stash.dynamic_reservation_settled_tokens - if increment <= 0: - return - previous: Final = stash.dynamic_reservation_settled_tokens - stash.dynamic_reservation_settled_tokens = actual_tokens - try: - await self.v3_limiter.async_increment_reservation_aware_tokens( - pipeline_operations=self.v3_limiter.build_project_reservation_ops( - targets=sorted(stash.dynamic_token_scopes), - reserved_scopes=frozenset(), - actual_tokens=increment, - reserved_tokens=0, - reservation_window_identities=stash.dynamic_reservation_windows, - ), - parent_otel_span=parent_otel_span, - ) - except Exception: - if stash.dynamic_reservation_settled_tokens == actual_tokens: + async with stash.dynamic_settlement_lock: + increment: Final = actual_tokens - stash.dynamic_reservation_settled_tokens + if increment <= 0: + return + previous: Final = stash.dynamic_reservation_settled_tokens + stash.dynamic_reservation_settled_tokens = actual_tokens + try: + await self.v3_limiter.async_increment_reservation_aware_tokens( + pipeline_operations=self.v3_limiter.build_project_reservation_ops( + targets=sorted(stash.dynamic_token_scopes), + reserved_scopes=frozenset(), + actual_tokens=increment, + reserved_tokens=0, + reservation_window_identities=stash.dynamic_reservation_windows, + ), + parent_otel_span=parent_otel_span, + ) + except Exception: stash.dynamic_reservation_settled_tokens = previous - raise + raise def _queue_weights(self, model_group_info: ModelGroupInfo, fairness: FairnessSettings) -> Mapping[str, float]: normalized: Final = self._normalize_priority_weights(model_group_info) diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 2964e9ffa34..8f277378ddc 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -592,6 +592,7 @@ class RequestRateLimiterStash: ) dynamic_reservation_settled: bool = False dynamic_reservation_settled_tokens: int = 0 + dynamic_settlement_lock: asyncio.Lock = field(default_factory=asyncio.Lock) dynamic_admission_bypassed: bool = False fairness_queue_wait_seconds: float = 0.0 diff --git a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py index 3dae84fcc20..49edc0f2d5e 100644 --- a/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py +++ b/tests/test_litellm/proxy/hooks/test_dynamic_rate_limiter_v3.py @@ -2755,7 +2755,7 @@ async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_to monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (6, 0, 0)) data = {"model": model, "litellm_call_id": "racing-settle"} - async def run_race() -> None: + async def run_race() -> tuple[bool, int]: stash = get_or_create_request_stash() stash.dynamic_reserved_tokens = 40 stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)}) @@ -2767,28 +2767,35 @@ async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_to user_api_key_dict=_prod_user(), ) ) - await asyncio.wait_for(first_call_seen.wait(), timeout=5) - await handler.async_log_success_event( - kwargs=_success_kwargs(model, "racing-settle", "prod"), - response_obj=ModelResponse( - model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) - ), - start_time=None, - end_time=None, + success_task = asyncio.create_task( + handler.async_log_success_event( + kwargs=_success_kwargs(model, "racing-settle", "prod"), + response_obj=ModelResponse( + model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) + ), + start_time=None, + end_time=None, + ) ) + await asyncio.wait_for(first_call_seen.wait(), timeout=5) + for _ in range(3): + await asyncio.sleep(0) settle_gate.set() - await failure_task + await asyncio.gather(failure_task, success_task) + return stash.dynamic_reservation_settled, stash.dynamic_reservation_settled_tokens - await asyncio.create_task(run_race()) + settled, settled_tokens = await asyncio.create_task(run_race()) assert len(calls) == 2 + first_ops = calls[0][1]["pipeline_operations"] + assert all(op["increment_value"] == -34 for op in first_ops) second_ops = calls[1][1]["pipeline_operations"] - assert len(second_ops) == 1 - for op in second_ops: - assert op["increment_value"] == 4 + assert all(op["increment_value"] == 4 for op in second_ops) + assert settled + assert settled_tokens == 10 @pytest.mark.asyncio -async def test_failed_settlement_write_keeps_the_record_of_a_concurrent_billing(monkeypatch): +async def test_failed_settlement_write_lets_the_waiting_success_settle_at_actual_usage(monkeypatch): from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass from litellm.types.utils import ModelResponse, Usage @@ -2828,20 +2835,26 @@ async def test_failed_settlement_write_keeps_the_record_of_a_concurrent_billing( user_api_key_dict=_prod_user(), ) ) - await asyncio.wait_for(first_call_seen.wait(), timeout=5) - await handler.async_log_success_event( - kwargs=_success_kwargs(model, "failed-settle-race", "prod"), - response_obj=ModelResponse( - model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) - ), - start_time=None, - end_time=None, + success_task = asyncio.create_task( + handler.async_log_success_event( + kwargs=_success_kwargs(model, "failed-settle-race", "prod"), + response_obj=ModelResponse( + model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10) + ), + start_time=None, + end_time=None, + ) ) + await asyncio.wait_for(first_call_seen.wait(), timeout=5) + for _ in range(3): + await asyncio.sleep(0) settle_gate.set() - await failure_task + await asyncio.gather(failure_task, success_task) return stash.dynamic_reservation_settled, stash.dynamic_reservation_settled_tokens settled, settled_tokens = await asyncio.create_task(run_race()) assert len(calls) == 2 + second_ops = calls[1][1]["pipeline_operations"] + assert all(op["increment_value"] == -30 for op in second_ops) assert settled assert settled_tokens == 10