fix(rate-limit): take a token hold before awaiting so concurrent success and failure callbacks settle it once

This commit is contained in:
mateo-berri 2026-10-31 16:59:13 -07:00
parent fb1ffe5ff5
commit 2aceca2871
2 changed files with 57 additions and 12 deletions

View file

@ -5575,18 +5575,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else None,
)
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
),
litellm_parent_otel_span,
usage_operations: Final = self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
)
io_token_operations: Final = self._build_io_token_reservation_ops(
kwargs=kwargs,
response_obj=response_obj,
)
if stash is not None:
stash.clear_reserved_tokens()
await self._apply_usage_operations(usage_operations, litellm_parent_otel_span)
if io_token_operations:
if isinstance(io_token_operations, list):
await self.async_increment_tokens_with_ttl_preservation(
@ -5598,8 +5598,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations=io_token_operations,
parent_otel_span=litellm_parent_otel_span,
)
if stash is not None:
stash.clear_reserved_tokens()
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
@ -5822,6 +5820,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else ()
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
if pipeline_operations:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call(
pipeline_operations, parent_otel_span=litellm_parent_otel_span
@ -5836,8 +5836,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations=project_operations,
parent_otel_span=litellm_parent_otel_span,
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
@ -6040,6 +6038,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
itpm_reserved,
otpm_reserved,
)
stash.reservation_released = True
if combined_ops:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
increment_list=list(combined_ops),
@ -6056,7 +6055,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
pipeline_operations=project_ops,
parent_otel_span=user_api_key_dict.parent_otel_span,
)
stash.reservation_released = True
except Exception as e:
verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e)
return

View file

@ -3746,6 +3746,53 @@ async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens
class _YieldingIncrementInMemoryCache(InMemoryCache):
async def async_increment(self, key: str, value: float, **kwargs: object) -> float:
await asyncio.sleep(0)
return await super().async_increment(key, value, **kwargs)
@pytest.mark.asyncio
@pytest.mark.parametrize("success_starts_first", [True, False])
@pytest.mark.parametrize("failure_path", ["failure_event", "post_call_failure_hook"])
async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_once_v3(
success_starts_first: bool, failure_path: str
) -> None:
api_key: Final = hash_token("sk-concurrent-failure-and-success")
local_cache: Final = DualCache(in_memory_cache=_YieldingIncrementInMemoryCache())
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens")
await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "concurrent-call")
kwargs: Final = {
"litellm_call_id": "concurrent-call",
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
}
now: Final = datetime.now()
log_failure: Final = (
partial(handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now)
if failure_path == "failure_event"
else partial(
handler.async_post_call_failure_hook,
request_data=dict(kwargs),
original_exception=Exception("stream ended"),
user_api_key_dict=UserAPIKeyAuth(api_key=api_key, tpm_limit=10_000),
)
)
log_success: Final = partial(
handler.async_log_success_event,
kwargs=dict(kwargs),
response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)),
start_time=now,
end_time=now,
)
await asyncio.gather(
*((log_success(), log_failure()) if success_starts_first else (log_failure(), log_success()))
)
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 30
@pytest.mark.asyncio
async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3():
"""