fix(accounting): share byte admission across batch producers and writes

This commit is contained in:
Joshua Valluru 2026-10-31 02:30:17 -07:00
parent e517e65bc2
commit a75be2bf78
4 changed files with 180 additions and 16 deletions

View file

@ -2,7 +2,8 @@ from __future__ import annotations
import asyncio
import json
from collections.abc import Awaitable, Callable, Mapping
import re
from collections.abc import Awaitable, Callable, Iterator, Mapping
from dataclasses import dataclass, replace
from datetime import datetime, timezone
from decimal import Decimal
@ -623,7 +624,7 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int)
class _BatchCapture:
payload: SpendLogsPayload
root: UsageWrite
rows: tuple[Mapping[str, JsonValue], ...]
rows: bytes
prerequisites: tuple[asyncio.Future[None], ...]
@ -641,25 +642,69 @@ def _prepare_batch_capture(
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
raise UsageCaptureError("Batch rows require one terminal job observation")
return _BatchCapture(
payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs))
payload,
_batch_root(builder.writes[0], rows),
b"\n".join(_json(row).encode("utf-8") for row in rows),
tuple(prior.receipt for prior in _prior_captures(kwargs)),
)
def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]:
return (
_batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index)
for index, match in enumerate(re.finditer(rb"[^\n]+", rows))
)
def _batch_write_bound(root: UsageWrite, rows: bytes) -> int:
return max(
len(root.envelope.encode("utf-8")),
max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)),
)
def _batch_capacity(
meter: DurableUsageMeter, root: UsageWrite, rows: bytes, *, extra_bytes: int = 0
) -> tuple[int, int, int]:
retained: Final = len(rows) + len(root.envelope.encode("utf-8")) + extra_bytes
bound: Final = _batch_write_bound(root, rows)
chunk_bytes: Final = max(
bound,
min(
bound * min(meter.settings.batch_size, rows.count(b"\n") + 1),
meter.settings.max_pending_bytes // 4,
(meter.settings.max_pending_bytes - retained) // 2,
),
)
return retained + chunk_bytes, chunk_bytes, bound
async def _persist_batch_capture(
meter: DurableUsageMeter,
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]],
capture: _BatchCapture,
root: UsageWrite,
rows: bytes,
prerequisites: tuple[asyncio.Future[None], ...],
chunk_bytes: int,
reservation: object | None = None,
) -> None:
for receipt in capture.prerequisites:
for receipt in prerequisites:
await asyncio.shield(receipt)
await meter.record_writes((capture.root,))
accepted: Final = await load((capture.root.event.event_id,))
await meter.record_writes((root,))
accepted: Final = await load((root.event.event_id,))
if len(accepted) != 1:
raise UsageCaptureError("Committed batch snapshot could not be loaded")
frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0])
remaining: Final = iter(enumerate(capture.rows))
while group := tuple(islice(remaining, meter.settings.batch_size)):
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
frozen: Final = _reuse_terminal_snapshot(root, accepted[0])
retained, capacity, bound = _batch_capacity(meter, frozen, rows, extra_bytes=len(root.envelope.encode("utf-8")))
if reservation is not None:
meter.resize_producer(reservation, retained, capacity)
write_capacity: Final = capacity if reservation is not None else chunk_bytes
if bound > write_capacity:
raise UsageCaptureError("Batch observation exceeds reserved producer write capacity")
group_size: Final = min(meter.settings.batch_size, max(1, write_capacity // bound))
remaining: Final = _batch_writes(frozen, rows)
while group := tuple(islice(remaining, group_size)):
await meter.record_writes(group)
async def capture_batch_callback(
@ -673,7 +718,9 @@ async def capture_batch_callback(
if load is None:
raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader")
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
await _persist_batch_capture(meter, load, capture)
await _persist_batch_capture(
meter, load, capture.root, capture.rows, capture.prerequisites, meter.settings.max_pending_bytes
)
receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
receipt.set_result(None)
return CapturedUsage(capture.payload, (capture.root,), receipt)
@ -986,7 +1033,15 @@ class UsageCaptureRuntime:
if len(self._batches) >= meter.settings.max_pending_records:
raise UsageCaptureError("Batch capture producer capacity exhausted")
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture")
retained, chunk_bytes, _ = _batch_capacity(meter, capture.root, capture.rows)
reservation: Final = meter.reserve_producer(retained, chunk_bytes)
task: Final = asyncio.create_task(
_persist_batch_capture(
meter, load, capture.root, capture.rows, capture.prerequisites, chunk_bytes, reservation
),
name="accounting batch capture",
)
task.add_done_callback(lambda _: meter.release_producer(reservation))
self._batches.add(task)
task.add_done_callback(self._batch_finished)
return CapturedUsage(capture.payload, (capture.root,), task)

View file

@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
from collections import deque
from collections.abc import Awaitable, Callable
from collections.abc import Awaitable, Callable, Mapping
from concurrent.futures import Future
from dataclasses import dataclass
from itertools import accumulate, chain, count, takewhile
@ -115,6 +115,8 @@ class DurableUsageMeter:
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
self._wakeup: Final = asyncio.Event()
self._capacity_changed: Final = asyncio.Event()
self._reservations: Mapping[object, tuple[int, int]] = {}
self._reserved_bytes = 0
self._task: asyncio.Task[None] | None = None
self._closed = False
self.pending_bytes = 0
@ -144,6 +146,36 @@ class DurableUsageMeter:
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
return wrapped
def reserve_producer(self, size: int, headroom: int) -> object:
token: Final = object()
self.resize_producer(token, size, headroom)
return token
def resize_producer(self, token: object, size: int, headroom: int) -> None:
with self._admission:
previous: Final = self._reservations.get(token, (0, 0))[0]
delta: Final = size - previous
others: Final = (value[1] for key, value in self._reservations.items() if key is not token)
required: Final = max((headroom, *others))
if (
self._closed
or self.pending_bytes + delta > self.settings.max_pending_bytes
or self._reserved_bytes + delta + required > self.settings.max_pending_bytes
):
raise UsageCaptureError("Batch capture producer byte capacity exhausted")
self._reservations = {**self._reservations, token: (size, headroom)}
self._reserved_bytes += delta
self.pending_bytes += delta
self._capacity_changed.set()
def release_producer(self, token: object) -> None:
with self._admission:
size, _ = self._reservations[token]
self._reservations = {key: value for key, value in self._reservations.items() if key is not token}
self._reserved_bytes -= size
self.pending_bytes -= size
self._capacity_changed.set()
def _ensure_worker(self) -> None:
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
self._wakeup.set()
@ -156,14 +188,15 @@ class DurableUsageMeter:
async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None:
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes
if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or (
size > self.settings.max_pending_bytes and len(writes) > 1
size > capacity and len(writes) > 1
):
middle: Final = len(writes) // 2
await self.record_writes(writes[:middle])
await self.record_writes(writes[middle:])
return
if size > self.settings.max_pending_bytes:
if size > capacity:
raise UsageCaptureError("One usage observation exceeds capture capacity")
while (receipt := self._submit_available(writes)) is None:
self._wakeup.set()

View file

@ -1607,3 +1607,70 @@ async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence(
release.set()
await asyncio.gather(callback, return_exceptions=True)
await runtime.stop(1)
@pytest.mark.asyncio
async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() -> None:
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
database: Final = StoredUsage()
release: Final = asyncio.Event()
runtime: Final = UsageCaptureRuntime()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
await release.wait()
assert runtime.meter is not None
assert runtime.meter.pending_bytes <= 65536
await database.insert(writes)
runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load)
rows: Final = tuple(
{"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None}
for index in range(1000)
)
response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}}
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
try:
first: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
assert runtime.meter is not None
assert 49000 < runtime.meter.pending_bytes <= 65536
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
release.set()
await asyncio.shield(first.receipt)
assert len(database.rows) == 1001
assert runtime.meter.pending_bytes == 0
second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
await asyncio.shield(second.receipt)
assert len(database.rows) == 2002
assert runtime.meter.pending_bytes == 0
finally:
release.set()
await runtime.stop(1)
@pytest.mark.asyncio
async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> None:
from litellm.proxy.spend_tracking.usage_capture import PricingCapture, UsageBatchWriter, UsageCaptureRuntime
database: Final = StoredUsage()
writer: Final = UsageBatchWriter(database.insert, database.load)
runtime: Final = UsageCaptureRuntime()
runtime.start(writer.write, UsageCaptureSettings(max_pending_bytes=16384), load=database.load)
response: Final = {
"id": "batch",
"status": "completed",
"usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},)},
}
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"model": {"source": "x" * 1200}}}))
try:
initial: Final = runtime.submit_batch({**kwargs, "_accounting_pricing_snapshot": pricing}, response, NOW, NOW)
await asyncio.shield(initial.receipt)
accepted: Final = database.rows
replay: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
await asyncio.shield(replay.receipt)
assert database.rows == accepted and len(accepted) == 2
assert runtime.meter is not None and runtime.meter.pending_bytes == 0
finally:
await runtime.stop(1)

View file

@ -78,6 +78,13 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N
meter: Final = DurableUsageMeter(
database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush
)
reservation: Final = meter.reserve_producer(size // 2, size // 2)
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
meter.reserve_producer(1, size)
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
meter.resize_producer(reservation, size, 1)
assert meter.pending_bytes == size // 2
meter.release_producer(reservation)
receipt: Final = meter.submit((write,))
with pytest.raises(UsageCaptureError, match="capacity"):
meter.submit((_write("extra"),))
@ -88,6 +95,8 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N
assert database.committed == (write.event.event_id,)
with pytest.raises(UsageCaptureError, match="closed"):
meter.submit((_write("after-close"),))
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
meter.reserve_producer(1, 1)
@pytest.mark.asyncio