mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(rate-limit): take a token hold before awaiting so concurrent success and failure callbacks settle it once
This commit is contained in:
parent
fb1ffe5ff5
commit
2aceca2871
2 changed files with 57 additions and 12 deletions
|
|
@ -5575,18 +5575,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else None,
|
||||
)
|
||||
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
),
|
||||
litellm_parent_otel_span,
|
||||
usage_operations: Final = self._build_success_event_pipeline_operations(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
rate_limit_type=rate_limit_type,
|
||||
)
|
||||
io_token_operations: Final = self._build_io_token_reservation_ops(
|
||||
kwargs=kwargs,
|
||||
response_obj=response_obj,
|
||||
)
|
||||
if stash is not None:
|
||||
stash.clear_reserved_tokens()
|
||||
await self._apply_usage_operations(usage_operations, litellm_parent_otel_span)
|
||||
if io_token_operations:
|
||||
if isinstance(io_token_operations, list):
|
||||
await self.async_increment_tokens_with_ttl_preservation(
|
||||
|
|
@ -5598,8 +5598,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
pipeline_operations=io_token_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
if stash is not None:
|
||||
stash.clear_reserved_tokens()
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
|
||||
|
|
@ -5822,6 +5820,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
else ()
|
||||
)
|
||||
|
||||
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
|
||||
stash.reservation_released = True
|
||||
if pipeline_operations:
|
||||
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call(
|
||||
pipeline_operations, parent_otel_span=litellm_parent_otel_span
|
||||
|
|
@ -5836,8 +5836,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
pipeline_operations=project_operations,
|
||||
parent_otel_span=litellm_parent_otel_span,
|
||||
)
|
||||
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
|
||||
stash.reservation_released = True
|
||||
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
|
||||
await self._apply_usage_operations(
|
||||
self._build_success_event_pipeline_operations(
|
||||
|
|
@ -6040,6 +6038,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
itpm_reserved,
|
||||
otpm_reserved,
|
||||
)
|
||||
stash.reservation_released = True
|
||||
if combined_ops:
|
||||
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
|
||||
increment_list=list(combined_ops),
|
||||
|
|
@ -6056,7 +6055,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
pipeline_operations=project_ops,
|
||||
parent_otel_span=user_api_key_dict.parent_otel_span,
|
||||
)
|
||||
stash.reservation_released = True
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception("Error releasing TPM reservation on post-call failure: %s", e)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -3746,6 +3746,53 @@ async def test_a_request_logging_a_failure_and_a_success_settles_its_reservation
|
|||
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == used_tokens
|
||||
|
||||
|
||||
class _YieldingIncrementInMemoryCache(InMemoryCache):
|
||||
async def async_increment(self, key: str, value: float, **kwargs: object) -> float:
|
||||
await asyncio.sleep(0)
|
||||
return await super().async_increment(key, value, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("success_starts_first", [True, False])
|
||||
@pytest.mark.parametrize("failure_path", ["failure_event", "post_call_failure_hook"])
|
||||
async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_once_v3(
|
||||
success_starts_first: bool, failure_path: str
|
||||
) -> None:
|
||||
api_key: Final = hash_token("sk-concurrent-failure-and-success")
|
||||
local_cache: Final = DualCache(in_memory_cache=_YieldingIncrementInMemoryCache())
|
||||
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
|
||||
tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens")
|
||||
await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "concurrent-call")
|
||||
kwargs: Final = {
|
||||
"litellm_call_id": "concurrent-call",
|
||||
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
|
||||
}
|
||||
now: Final = datetime.now()
|
||||
log_failure: Final = (
|
||||
partial(handler.async_log_failure_event, kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now)
|
||||
if failure_path == "failure_event"
|
||||
else partial(
|
||||
handler.async_post_call_failure_hook,
|
||||
request_data=dict(kwargs),
|
||||
original_exception=Exception("stream ended"),
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key=api_key, tpm_limit=10_000),
|
||||
)
|
||||
)
|
||||
log_success: Final = partial(
|
||||
handler.async_log_success_event,
|
||||
kwargs=dict(kwargs),
|
||||
response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=10, total_tokens=30)),
|
||||
start_time=now,
|
||||
end_time=now,
|
||||
)
|
||||
|
||||
await asyncio.gather(
|
||||
*((log_success(), log_failure()) if success_starts_first else (log_failure(), log_success()))
|
||||
)
|
||||
|
||||
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 30
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_call_failure_hook_settles_reservation_at_recovered_partial_usage_v3():
|
||||
"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue