fix(rate-limit): track the ITPM/OTPM hold's release apart from TPM so a nested pre-call keeps the request's project IO charge

This commit is contained in:
mateo-berri 2026-11-02 15:47:08 -08:00
parent 2aceca2871
commit 07cca4dbf0
3 changed files with 134 additions and 30 deletions

View file

@ -709,7 +709,7 @@ class RequestRateLimiterStash:
A single mutable instance is shared by every context forked from the
request task (the SDK call, streaming generators, and the logging worker's
captured context all see the same object), which is what makes the
``reservation_released`` flag and ``parallel_slot`` clearing effective
released flags and ``parallel_slot`` clearing effective
across sibling callbacks: the first release wins, later callbacks observe
the cleared state.
@ -746,6 +746,7 @@ class RequestRateLimiterStash:
batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
reservation_released: bool = False
io_reservation_released: bool = False
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
@ -755,11 +756,22 @@ class RequestRateLimiterStash:
self.itpm_reserved_tokens = 0
self.otpm_reserved_tokens = 0
def discard_released_reservation(self) -> None:
def mark_reservation_released(self) -> None:
self.reservation_released = True
self.io_reservation_released = True
def discard_released_tpm_reservation(self) -> None:
if self.reservation_released:
self.clear_reserved_tokens()
self.reserved_tokens = 0
self.ptu_reserved_tokens = 0
self.reservation_released = False
def discard_released_io_reservation(self) -> None:
if self.io_reservation_released:
self.itpm_reserved_tokens = 0
self.otpm_reserved_tokens = 0
self.io_reservation_released = False
@dataclass(frozen=True, slots=True)
class CounterRefund:
@ -4249,7 +4261,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
requested_model=requested_model,
)
stash.discard_released_reservation()
stash.discard_released_io_reservation()
if itpm_reserved > 0:
itpm_scopes: Final = tuple(
(d["key"], d["value"]) for d in io_token_descriptors if d["key"] == PROJECT_ITPM_DESCRIPTOR_KEY
@ -4609,7 +4621,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# incremented so post-call reconciliation only applies
# the (actual - reserved) delta to those — unreserved
# scopes get charged the full actual usage instead.
stash.discard_released_reservation()
stash.discard_released_tpm_reservation()
stash.reserved_tokens = estimated_tokens
stash.ptu_reserved_tokens = ptu_estimated_tokens
stash.ptu_reserved_window_identities = frozenset(
@ -5153,7 +5165,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
self._response_usage(response_obj)
) or self._aggregate_only_total_tokens(self._response_usage(kwargs.get("combined_usage_object")))
if not response_usage[2] and not combined_usage[2] and aggregate_total <= 0 and not stash.reservation_released:
if (
not response_usage[2]
and not combined_usage[2]
and aggregate_total <= 0
and not stash.io_reservation_released
):
return ()
resolved_usage: Final = (
response_usage
@ -5166,25 +5183,25 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
billable_input, completion_tokens, _ = resolved_usage
if stash.reservation_released or (
if stash.io_reservation_released or (
not stash.itpm_reserved_window_identities and not stash.otpm_reserved_window_identities
):
return self._build_reservation_aware_tpm_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes,
actual_tokens=billable_input,
reserved_tokens=0 if stash.reservation_released else itpm_reserved,
reserved_tokens=0 if stash.io_reservation_released else itpm_reserved,
) + self._build_reservation_aware_tpm_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes,
actual_tokens=completion_tokens,
reserved_tokens=0 if stash.reservation_released else otpm_reserved,
reserved_tokens=0 if stash.io_reservation_released else otpm_reserved,
)
itpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
self._build_project_reservation_ops(
targets=tuple(stash.itpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.itpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.itpm_reserved_scopes,
actual_tokens=billable_input,
reserved_tokens=itpm_reserved,
reservation_window_identities=stash.itpm_reserved_window_identities,
@ -5195,7 +5212,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
otpm_ops: Final[Sequence[ReservationAwareIncrementOperation]] = (
self._build_project_reservation_ops(
targets=tuple(stash.otpm_reserved_scopes),
reserved_scopes=frozenset() if stash.reservation_released else stash.otpm_reserved_scopes,
reserved_scopes=frozenset() if stash.io_reservation_released else stash.otpm_reserved_scopes,
actual_tokens=completion_tokens,
reserved_tokens=otpm_reserved,
reservation_window_identities=stash.otpm_reserved_window_identities,
@ -5747,10 +5764,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# already released it (proxy-level rejection that also bubbles up
# here as an LLM-error callback). max_parallel_requests is its
# own counter and is always decremented per call.
reserved_tokens, itpm_reserved, otpm_reserved = (
(0, 0, 0)
if stash is None or stash.reservation_released
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
reserved_tokens: Final = 0 if stash is None or stash.reservation_released else stash.reserved_tokens
itpm_reserved, otpm_reserved = (
(0, 0)
if stash is None or stash.io_reservation_released
else (stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
@ -5821,7 +5839,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
stash.mark_reservation_released()
if pipeline_operations:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline_post_call(
pipeline_operations, parent_otel_span=litellm_parent_otel_span
@ -5941,7 +5959,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
Idempotent: the slot release clears the stashed acquisition (and slot
removal is a no-op ZREM on a second run), and the TPM/ITPM/OTPM
refund is guarded by the stash's ``reservation_released`` flag — if
refund is guarded by the stash's two released flags — if
both this hook and async_log_failure_event end up running in the same
flow, only the first release/refund applies. A mid-stream failure
relayed here with recovered partial usage settles the reservation at
@ -5967,11 +5985,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
stash.batch_tpd_refund_ops = ()
if stash.reservation_released:
return
reserved_tokens: Final = stash.reserved_tokens
itpm_reserved: Final = stash.itpm_reserved_tokens
otpm_reserved: Final = stash.otpm_reserved_tokens
reserved_tokens: Final = 0 if stash.reservation_released else stash.reserved_tokens
itpm_reserved: Final = 0 if stash.io_reservation_released else stash.itpm_reserved_tokens
otpm_reserved: Final = 0 if stash.io_reservation_released else stash.otpm_reserved_tokens
if reserved_tokens <= 0 and itpm_reserved <= 0 and otpm_reserved <= 0:
return
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(request_data)
@ -6038,7 +6054,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
itpm_reserved,
otpm_reserved,
)
stash.reservation_released = True
stash.mark_reservation_released()
if combined_ops:
await self.internal_usage_cache.dual_cache.async_increment_cache_pipeline(
increment_list=list(combined_ops),

View file

@ -3759,12 +3759,39 @@ async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_onc
success_starts_first: bool, failure_path: str
) -> None:
api_key: Final = hash_token("sk-concurrent-failure-and-success")
model: Final = "gpt-4o-mini"
project: Final = "proj-concurrent"
local_cache: Final = DualCache(in_memory_cache=_YieldingIncrementInMemoryCache())
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
tokens_key: Final = handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens")
await _reserve_tpm_for_owner_call(handler, local_cache, api_key, "concurrent-call")
caller: Final = UserAPIKeyAuth(
api_key=api_key,
tpm_limit=10_000,
project_id=project,
project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}},
)
counter_keys: Final = (
handler.create_rate_limit_keys(key="api_key", value=api_key, rate_limit_type="tokens"),
handler.create_rate_limit_keys(
key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens"
),
handler.create_rate_limit_keys(
key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens"
),
)
await handler.async_pre_call_hook(
user_api_key_dict=caller,
cache=local_cache,
data={
"model": model,
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 50,
"litellm_call_id": "concurrent-call",
},
call_type="completion",
)
kwargs: Final = {
"litellm_call_id": "concurrent-call",
"model": model,
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
}
now: Final = datetime.now()
@ -3775,7 +3802,7 @@ async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_onc
handler.async_post_call_failure_hook,
request_data=dict(kwargs),
original_exception=Exception("stream ended"),
user_api_key_dict=UserAPIKeyAuth(api_key=api_key, tpm_limit=10_000),
user_api_key_dict=caller,
)
)
log_success: Final = partial(
@ -3790,7 +3817,68 @@ async def test_concurrent_failure_and_success_callbacks_settle_a_reservation_onc
*((log_success(), log_failure()) if success_starts_first else (log_failure(), log_success()))
)
assert int(await local_cache.async_get_cache(key=tokens_key) or 0) == 30
assert tuple([int(await local_cache.async_get_cache(key=key) or 0) for key in counter_keys]) == (30, 20, 10)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("second_call_type", "second_model"),
[("call_mcp_tool", "mcp-tool-call"), ("completion", "gpt-4o-mini")],
)
async def test_a_pre_call_after_a_failure_leaves_the_success_charging_project_io_once_v3(
second_call_type: str, second_model: str
) -> None:
api_key: Final = hash_token("sk-retried-request-with-mcp-tool")
model: Final = "gpt-4o-mini"
project: Final = "proj-retried-mcp"
local_cache: Final = DualCache()
handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(local_cache))
caller: Final = UserAPIKeyAuth(
api_key=api_key,
tpm_limit=100_000,
project_id=project,
project_metadata={"model_itpm_limit": {model: 10_000}, "model_otpm_limit": {model: 10_000}},
)
request: Final = {
"model": model,
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 50,
"litellm_call_id": "retried-call-with-mcp-tool",
}
kwargs: Final = {
"litellm_call_id": "retried-call-with-mcp-tool",
"model": model,
"standard_logging_object": {"metadata": {"user_api_key_hash": api_key}},
}
now: Final = datetime.now()
itpm_key: Final = handler.create_rate_limit_keys(
key="model_per_project_itpm", value=f"{project}:{model}", rate_limit_type="tokens"
)
otpm_key: Final = handler.create_rate_limit_keys(
key="model_per_project_otpm", value=f"{project}:{model}", rate_limit_type="tokens"
)
await handler.async_pre_call_hook(
user_api_key_dict=caller, cache=local_cache, data=dict(request), call_type="completion"
)
await handler.async_log_failure_event(kwargs=dict(kwargs), response_obj=None, start_time=now, end_time=now)
await handler.async_pre_call_hook(
user_api_key_dict=caller,
cache=local_cache,
data={**request, "model": second_model},
call_type=second_call_type,
)
await handler.async_log_success_event(
kwargs=dict(kwargs),
response_obj=ModelResponse(usage=Usage(prompt_tokens=20, completion_tokens=7, total_tokens=27)),
start_time=now,
end_time=now,
)
assert (
int(await local_cache.async_get_cache(key=itpm_key) or 0),
int(await local_cache.async_get_cache(key=otpm_key) or 0),
) == (20, 7)
@pytest.mark.asyncio

View file

@ -3060,7 +3060,7 @@ def test_retry_success_charges_released_project_io_reservations(
stash.itpm_reserved_scopes = frozenset({itpm_scope})
stash.otpm_reserved_tokens = 60
stash.otpm_reserved_scopes = frozenset({otpm_scope})
stash.reservation_released = True
stash.mark_reservation_released()
operations = handler._build_io_token_reservation_ops(
{},