From f2b01eabc8e40a9e37b2af1f8951d67498870f62 Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 20:10:25 -0700 Subject: [PATCH] fix(accounting): await commit prerequisites within bounded usage admission --- litellm/proxy/spend_tracking/usage_meter.py | 28 +++++++++-- .../proxy/spend_tracking/test_usage_meter.py | 48 +++++++++++++++++++ 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py index 98e626ff82d..3bc7572b476 100644 --- a/litellm/proxy/spend_tracking/usage_meter.py +++ b/litellm/proxy/spend_tracking/usage_meter.py @@ -58,6 +58,7 @@ class _Pending: size: int committed: Future[None] finished: asyncio.Event + prerequisites: tuple[asyncio.Future[None], ...] async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None: @@ -118,7 +119,9 @@ class DurableUsageMeter: self.pending_bytes = 0 self.pending_records = 0 - def submit(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None]: + def submit( + self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = () + ) -> asyncio.Future[None]: size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) with self._admission: if ( @@ -132,7 +135,7 @@ class DurableUsageMeter: receipt: Final = Future[None]() receipt.set_running_or_notify_cancel() receipt.add_done_callback(_report_failure) - self._queue.append(_Pending(writes, size, receipt, asyncio.Event())) + self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites)) self.pending_bytes += size self.pending_records += len(writes) self._loop.call_soon_threadsafe(self._ensure_worker) @@ -141,7 +144,7 @@ class DurableUsageMeter: return wrapped def _ensure_worker(self) -> None: - if self.pending_records >= self.settings.batch_size: + if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites: self._wakeup.set() if self._task is None or self._task.done(): self._task = asyncio.create_task(self._run(), name="accounting usage capture") @@ -178,7 +181,10 @@ class DurableUsageMeter: def _take_batch(self) -> tuple[_Pending, ...]: with self._admission: - sizes: Final = accumulate(len(item.writes) for item in self._queue) + if self._queue[0].prerequisites: + return (self._queue.popleft(),) + independent: Final = takewhile(lambda item: not item.prerequisites, self._queue) + sizes: Final = accumulate(len(item.writes) for item in independent) count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes)) return tuple(self._queue.popleft() for _ in range(count)) @@ -197,6 +203,13 @@ class DurableUsageMeter: async def _commit(self, batch: tuple[_Pending, ...]) -> None: writes: Final = tuple(chain.from_iterable(item.writes for item in batch)) try: + if batch[0].prerequisites: + outcomes: Final = await asyncio.gather( + *(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True + ) + for outcome in outcomes: + if isinstance(outcome, BaseException): + raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome await self._persist(writes) except asyncio.CancelledError: self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted")) @@ -240,7 +253,12 @@ class DurableUsageMeter: await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds) self._wakeup.clear() await self._commit(self._take_batch()) - if self._closed or self.pending_records >= self.settings.batch_size: + if ( + self._closed + or self.pending_records >= self.settings.batch_size + or self._queue + and self._queue[0].prerequisites + ): self._wakeup.set() except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally while self._queue: diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py index e4584e4522e..de7d7c6591d 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_meter.py +++ b/tests/unit/proxy/spend_tracking/test_usage_meter.py @@ -339,3 +339,51 @@ async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqls finally: await meter.close() assert (meter.pending_records, meter.pending_bytes) == (0, 0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dependency_outcome", ["commit", "failure", "cancel"]) +async def test_commit_dependencies_preserve_capacity_and_isolate_failed_dependents(dependency_outcome: str) -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3, max_pending_records=4)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + neighbor: Final = meter.submit((_write("neighbor"),)) + assert meter.pending_records == 3 + assert not dependent.done() + with pytest.raises(UsageCaptureError, match="capacity"): + meter.submit((_write("excess-one"), _write("excess-two"))) + if dependency_outcome == "commit": + prerequisite.set_result(None) + elif dependency_outcome == "failure": + prerequisite.set_exception(ValueError("dependency failed")) + else: + prerequisite.cancel() + database.release.set() + await meter.close() + await neighbor + if dependency_outcome == "commit": + await dependent + assert database.committed == ("parent", "final", "neighbor") + else: + with pytest.raises(UsageCaptureError, match="dependency"): + await dependent + assert database.committed == ("neighbor",) + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + + +@pytest.mark.asyncio +async def test_shutdown_does_not_cancel_dependency_receipt_or_leak_reserved_capacity() -> None: + database: Final = CommitBarrier() + meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2)) + prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() + dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,)) + await meter.stop(0) + with pytest.raises(UsageCaptureError, match="acknowledgement"): + await dependent + assert not prerequisite.done() + assert database.committed == () + assert meter.pending_records == 0 + assert meter.pending_bytes == 0 + prerequisite.set_result(None)