fix(accounting): await commit prerequisites within bounded usage admission

This commit is contained in:
Joshua Valluru 2026-10-30 20:10:25 -07:00
parent 4ff4538bac
commit f2b01eabc8
2 changed files with 71 additions and 5 deletions

View file

@ -58,6 +58,7 @@ class _Pending:
size: int
committed: Future[None]
finished: asyncio.Event
prerequisites: tuple[asyncio.Future[None], ...]
async def _wait_for_batch(wakeup: asyncio.Event, timeout: float) -> None:
@ -118,7 +119,9 @@ class DurableUsageMeter:
self.pending_bytes = 0
self.pending_records = 0
def submit(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None]:
def submit(
self, writes: tuple[UsageWrite, ...], *, prerequisites: tuple[asyncio.Future[None], ...] = ()
) -> asyncio.Future[None]:
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
with self._admission:
if (
@ -132,7 +135,7 @@ class DurableUsageMeter:
receipt: Final = Future[None]()
receipt.set_running_or_notify_cancel()
receipt.add_done_callback(_report_failure)
self._queue.append(_Pending(writes, size, receipt, asyncio.Event()))
self._queue.append(_Pending(writes, size, receipt, asyncio.Event(), prerequisites))
self.pending_bytes += size
self.pending_records += len(writes)
self._loop.call_soon_threadsafe(self._ensure_worker)
@ -141,7 +144,7 @@ class DurableUsageMeter:
return wrapped
def _ensure_worker(self) -> None:
if self.pending_records >= self.settings.batch_size:
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
self._wakeup.set()
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._run(), name="accounting usage capture")
@ -178,7 +181,10 @@ class DurableUsageMeter:
def _take_batch(self) -> tuple[_Pending, ...]:
with self._admission:
sizes: Final = accumulate(len(item.writes) for item in self._queue)
if self._queue[0].prerequisites:
return (self._queue.popleft(),)
independent: Final = takewhile(lambda item: not item.prerequisites, self._queue)
sizes: Final = accumulate(len(item.writes) for item in independent)
count: Final = sum(1 for _ in takewhile(lambda size: size <= self.settings.batch_size, sizes))
return tuple(self._queue.popleft() for _ in range(count))
@ -197,6 +203,13 @@ class DurableUsageMeter:
async def _commit(self, batch: tuple[_Pending, ...]) -> None:
writes: Final = tuple(chain.from_iterable(item.writes for item in batch))
try:
if batch[0].prerequisites:
outcomes: Final = await asyncio.gather(
*(asyncio.shield(receipt) for receipt in batch[0].prerequisites), return_exceptions=True
)
for outcome in outcomes:
if isinstance(outcome, BaseException):
raise UsageCaptureError("Usage dependency did not acknowledge commit") from outcome
await self._persist(writes)
except asyncio.CancelledError:
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
@ -240,7 +253,12 @@ class DurableUsageMeter:
await self._wait_for_batch(self._wakeup, self.settings.flush_interval_seconds)
self._wakeup.clear()
await self._commit(self._take_batch())
if self._closed or self.pending_records >= self.settings.batch_size:
if (
self._closed
or self.pending_records >= self.settings.batch_size
or self._queue
and self._queue[0].prerequisites
):
self._wakeup.set()
except (Exception, asyncio.CancelledError): # noqa: BLE001 # reject queued receipts if the writer exits abnormally
while self._queue:

View file

@ -339,3 +339,51 @@ async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqls
finally:
await meter.close()
assert (meter.pending_records, meter.pending_bytes) == (0, 0)
@pytest.mark.asyncio
@pytest.mark.parametrize("dependency_outcome", ["commit", "failure", "cancel"])
async def test_commit_dependencies_preserve_capacity_and_isolate_failed_dependents(dependency_outcome: str) -> None:
database: Final = CommitBarrier()
meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=3, max_pending_records=4))
prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,))
neighbor: Final = meter.submit((_write("neighbor"),))
assert meter.pending_records == 3
assert not dependent.done()
with pytest.raises(UsageCaptureError, match="capacity"):
meter.submit((_write("excess-one"), _write("excess-two")))
if dependency_outcome == "commit":
prerequisite.set_result(None)
elif dependency_outcome == "failure":
prerequisite.set_exception(ValueError("dependency failed"))
else:
prerequisite.cancel()
database.release.set()
await meter.close()
await neighbor
if dependency_outcome == "commit":
await dependent
assert database.committed == ("parent", "final", "neighbor")
else:
with pytest.raises(UsageCaptureError, match="dependency"):
await dependent
assert database.committed == ("neighbor",)
assert meter.pending_records == 0
assert meter.pending_bytes == 0
@pytest.mark.asyncio
async def test_shutdown_does_not_cancel_dependency_receipt_or_leak_reserved_capacity() -> None:
database: Final = CommitBarrier()
meter: Final = DurableUsageMeter(database.write, UsageCaptureSettings(batch_size=2))
prerequisite: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
dependent: Final = meter.submit((_write("parent"), _write("final")), prerequisites=(prerequisite,))
await meter.stop(0)
with pytest.raises(UsageCaptureError, match="acknowledgement"):
await dependent
assert not prerequisite.done()
assert database.committed == ()
assert meter.pending_records == 0
assert meter.pending_bytes == 0
prerequisite.set_result(None)