mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
fix(proxy): keep a concurrent billing record when the reservation write fails
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
parent
a0e7aba7f4
commit
6aff9dbd9b
2 changed files with 65 additions and 3 deletions
|
|
@ -645,8 +645,9 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
except Exception:
|
||||
stash.dynamic_reservation_settled = False
|
||||
stash.dynamic_reservation_settled_tokens = 0
|
||||
if stash.dynamic_reservation_settled_tokens == actual_tokens:
|
||||
stash.dynamic_reservation_settled = False
|
||||
stash.dynamic_reservation_settled_tokens = 0
|
||||
raise
|
||||
return True
|
||||
|
||||
|
|
@ -673,7 +674,8 @@ class _PROXY_DynamicRateLimitHandlerV3(CustomLogger):
|
|||
parent_otel_span=parent_otel_span,
|
||||
)
|
||||
except Exception:
|
||||
stash.dynamic_reservation_settled_tokens = previous
|
||||
if stash.dynamic_reservation_settled_tokens == actual_tokens:
|
||||
stash.dynamic_reservation_settled_tokens = previous
|
||||
raise
|
||||
|
||||
def _queue_weights(self, model_group_info: ModelGroupInfo, fairness: FairnessSettings) -> Mapping[str, float]:
|
||||
|
|
|
|||
|
|
@ -2785,3 +2785,63 @@ async def test_success_racing_an_inflight_settlement_bills_only_the_uncounted_to
|
|||
assert len(second_ops) == 1
|
||||
for op in second_ops:
|
||||
assert op["increment_value"] == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_settlement_write_keeps_the_record_of_a_concurrent_billing(monkeypatch):
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.types.proxy.fairness import FairnessSettings, WorkloadClass
|
||||
from litellm.types.utils import ModelResponse, Usage
|
||||
|
||||
model = "fairness-failed-settle-race-model"
|
||||
_enable_fairness(
|
||||
monkeypatch,
|
||||
FairnessSettings(enabled=True, workload_classes=(WorkloadClass(name="prod", reserved_share=0.5),)),
|
||||
)
|
||||
dual_cache = DualCache()
|
||||
handler = DynamicRateLimitHandler(internal_usage_cache=dual_cache)
|
||||
handler.update_variables(llm_router=_fairness_router(model, tpm=100_000))
|
||||
settle_gate = asyncio.Event()
|
||||
first_call_seen = asyncio.Event()
|
||||
calls: list[tuple] = []
|
||||
|
||||
async def gated_increment(*args, **kwargs):
|
||||
calls.append((args, kwargs))
|
||||
if len(calls) == 1:
|
||||
first_call_seen.set()
|
||||
await settle_gate.wait()
|
||||
raise RuntimeError("redis unavailable")
|
||||
|
||||
monkeypatch.setattr(handler.v3_limiter, "async_increment_reservation_aware_tokens", gated_increment)
|
||||
monkeypatch.setattr(handler.v3_limiter, "recovered_partial_usage_tokens", lambda *args, **kwargs: (6, 0, 0))
|
||||
data = {"model": model, "litellm_call_id": "failed-settle-race"}
|
||||
|
||||
async def run_race() -> tuple[bool, int]:
|
||||
stash = get_or_create_request_stash()
|
||||
stash.dynamic_reserved_tokens = 40
|
||||
stash.dynamic_token_scopes = frozenset({("model_saturation_check", model)})
|
||||
|
||||
failure_task = asyncio.create_task(
|
||||
handler.async_post_call_failure_hook(
|
||||
request_data=data,
|
||||
original_exception=Exception("post-call guardrail rejected"),
|
||||
user_api_key_dict=_prod_user(),
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(first_call_seen.wait(), timeout=5)
|
||||
await handler.async_log_success_event(
|
||||
kwargs=_success_kwargs(model, "failed-settle-race", "prod"),
|
||||
response_obj=ModelResponse(
|
||||
model=model, usage=Usage(prompt_tokens=5, completion_tokens=5, total_tokens=10)
|
||||
),
|
||||
start_time=None,
|
||||
end_time=None,
|
||||
)
|
||||
settle_gate.set()
|
||||
await failure_task
|
||||
return stash.dynamic_reservation_settled, stash.dynamic_reservation_settled_tokens
|
||||
|
||||
settled, settled_tokens = await asyncio.create_task(run_race())
|
||||
assert len(calls) == 2
|
||||
assert settled
|
||||
assert settled_tokens == 10
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue