diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index a8e76505947..0db2a93ec87 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -2,7 +2,8 @@ from __future__ import annotations import asyncio import json -from collections.abc import Awaitable, Callable, Mapping +import re +from collections.abc import Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass, replace from datetime import datetime, timezone from decimal import Decimal @@ -623,7 +624,7 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int) class _BatchCapture: payload: SpendLogsPayload root: UsageWrite - rows: tuple[Mapping[str, JsonValue], ...] + rows: bytes prerequisites: tuple[asyncio.Future[None], ...] @@ -641,25 +642,69 @@ def _prepare_batch_capture( if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None: raise UsageCaptureError("Batch rows require one terminal job observation") return _BatchCapture( - payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs)) + payload, + _batch_root(builder.writes[0], rows), + b"\n".join(_json(row).encode("utf-8") for row in rows), + tuple(prior.receipt for prior in _prior_captures(kwargs)), ) +def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]: + return ( + _batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index) + for index, match in enumerate(re.finditer(rb"[^\n]+", rows)) + ) + + +def _batch_write_bound(root: UsageWrite, rows: bytes) -> int: + return max( + len(root.envelope.encode("utf-8")), + max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)), + ) + + +def _batch_capacity( + meter: DurableUsageMeter, root: UsageWrite, rows: bytes, *, extra_bytes: int = 0 +) -> tuple[int, int, int]: + retained: Final = len(rows) + len(root.envelope.encode("utf-8")) + extra_bytes + bound: Final = _batch_write_bound(root, rows) + chunk_bytes: Final = max( + bound, + min( + bound * min(meter.settings.batch_size, rows.count(b"\n") + 1), + meter.settings.max_pending_bytes // 4, + (meter.settings.max_pending_bytes - retained) // 2, + ), + ) + return retained + chunk_bytes, chunk_bytes, bound + + async def _persist_batch_capture( meter: DurableUsageMeter, load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]], - capture: _BatchCapture, + root: UsageWrite, + rows: bytes, + prerequisites: tuple[asyncio.Future[None], ...], + chunk_bytes: int, + reservation: object | None = None, ) -> None: - for receipt in capture.prerequisites: + for receipt in prerequisites: await asyncio.shield(receipt) - await meter.record_writes((capture.root,)) - accepted: Final = await load((capture.root.event.event_id,)) + await meter.record_writes((root,)) + accepted: Final = await load((root.event.event_id,)) if len(accepted) != 1: raise UsageCaptureError("Committed batch snapshot could not be loaded") - frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0]) - remaining: Final = iter(enumerate(capture.rows)) - while group := tuple(islice(remaining, meter.settings.batch_size)): - await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group)) + frozen: Final = _reuse_terminal_snapshot(root, accepted[0]) + retained, capacity, bound = _batch_capacity(meter, frozen, rows, extra_bytes=len(root.envelope.encode("utf-8"))) + if reservation is not None: + meter.resize_producer(reservation, retained, capacity) + write_capacity: Final = capacity if reservation is not None else chunk_bytes + if bound > write_capacity: + raise UsageCaptureError("Batch observation exceeds reserved producer write capacity") + group_size: Final = min(meter.settings.batch_size, max(1, write_capacity // bound)) + remaining: Final = _batch_writes(frozen, rows) + while group := tuple(islice(remaining, group_size)): + await meter.record_writes(group) async def capture_batch_callback( @@ -673,7 +718,9 @@ async def capture_batch_callback( if load is None: raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader") capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) - await _persist_batch_capture(meter, load, capture) + await _persist_batch_capture( + meter, load, capture.root, capture.rows, capture.prerequisites, meter.settings.max_pending_bytes + ) receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() receipt.set_result(None) return CapturedUsage(capture.payload, (capture.root,), receipt) @@ -986,7 +1033,15 @@ class UsageCaptureRuntime: if len(self._batches) >= meter.settings.max_pending_records: raise UsageCaptureError("Batch capture producer capacity exhausted") capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) - task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture") + retained, chunk_bytes, _ = _batch_capacity(meter, capture.root, capture.rows) + reservation: Final = meter.reserve_producer(retained, chunk_bytes) + task: Final = asyncio.create_task( + _persist_batch_capture( + meter, load, capture.root, capture.rows, capture.prerequisites, chunk_bytes, reservation + ), + name="accounting batch capture", + ) + task.add_done_callback(lambda _: meter.release_producer(reservation)) self._batches.add(task) task.add_done_callback(self._batch_finished) return CapturedUsage(capture.payload, (capture.root,), task) diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py index c776facfff6..431e4c2c3c3 100644 --- a/litellm/proxy/spend_tracking/usage_meter.py +++ b/litellm/proxy/spend_tracking/usage_meter.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio from collections import deque -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from concurrent.futures import Future from dataclasses import dataclass from itertools import accumulate, chain, count, takewhile @@ -115,6 +115,8 @@ class DurableUsageMeter: self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue self._wakeup: Final = asyncio.Event() self._capacity_changed: Final = asyncio.Event() + self._reservations: Mapping[object, tuple[int, int]] = {} + self._reserved_bytes = 0 self._task: asyncio.Task[None] | None = None self._closed = False self.pending_bytes = 0 @@ -144,6 +146,36 @@ class DurableUsageMeter: wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception()) return wrapped + def reserve_producer(self, size: int, headroom: int) -> object: + token: Final = object() + self.resize_producer(token, size, headroom) + return token + + def resize_producer(self, token: object, size: int, headroom: int) -> None: + with self._admission: + previous: Final = self._reservations.get(token, (0, 0))[0] + delta: Final = size - previous + others: Final = (value[1] for key, value in self._reservations.items() if key is not token) + required: Final = max((headroom, *others)) + if ( + self._closed + or self.pending_bytes + delta > self.settings.max_pending_bytes + or self._reserved_bytes + delta + required > self.settings.max_pending_bytes + ): + raise UsageCaptureError("Batch capture producer byte capacity exhausted") + self._reservations = {**self._reservations, token: (size, headroom)} + self._reserved_bytes += delta + self.pending_bytes += delta + self._capacity_changed.set() + + def release_producer(self, token: object) -> None: + with self._admission: + size, _ = self._reservations[token] + self._reservations = {key: value for key, value in self._reservations.items() if key is not token} + self._reserved_bytes -= size + self.pending_bytes -= size + self._capacity_changed.set() + def _ensure_worker(self) -> None: if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites: self._wakeup.set() @@ -156,14 +188,15 @@ class DurableUsageMeter: async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None: size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes) + capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or ( - size > self.settings.max_pending_bytes and len(writes) > 1 + size > capacity and len(writes) > 1 ): middle: Final = len(writes) // 2 await self.record_writes(writes[:middle]) await self.record_writes(writes[middle:]) return - if size > self.settings.max_pending_bytes: + if size > capacity: raise UsageCaptureError("One usage observation exceeds capture capacity") while (receipt := self._submit_available(writes)) is None: self._wakeup.set() diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index c89f1bd91d4..a1a165b03ef 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1607,3 +1607,70 @@ async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence( release.set() await asyncio.gather(callback, return_exceptions=True) await runtime.stop(1) + + +@pytest.mark.asyncio +async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() -> None: + from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime + + database: Final = StoredUsage() + release: Final = asyncio.Event() + runtime: Final = UsageCaptureRuntime() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + await release.wait() + assert runtime.meter is not None + assert runtime.meter.pending_bytes <= 65536 + await database.insert(writes) + + runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load) + rows: Final = tuple( + {"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None} + for index in range(1000) + ) + response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}} + kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} + try: + first: Final = runtime.submit_batch(kwargs, response, NOW, NOW) + assert runtime.meter is not None + assert 49000 < runtime.meter.pending_bytes <= 65536 + with pytest.raises(UsageCaptureError, match="producer byte capacity"): + runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW) + release.set() + await asyncio.shield(first.receipt) + assert len(database.rows) == 1001 + assert runtime.meter.pending_bytes == 0 + second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW) + await asyncio.shield(second.receipt) + assert len(database.rows) == 2002 + assert runtime.meter.pending_bytes == 0 + finally: + release.set() + await runtime.stop(1) + + +@pytest.mark.asyncio +async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> None: + from litellm.proxy.spend_tracking.usage_capture import PricingCapture, UsageBatchWriter, UsageCaptureRuntime + + database: Final = StoredUsage() + writer: Final = UsageBatchWriter(database.insert, database.load) + runtime: Final = UsageCaptureRuntime() + runtime.start(writer.write, UsageCaptureSettings(max_pending_bytes=16384), load=database.load) + response: Final = { + "id": "batch", + "status": "completed", + "usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},)}, + } + kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} + pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"model": {"source": "x" * 1200}}})) + try: + initial: Final = runtime.submit_batch({**kwargs, "_accounting_pricing_snapshot": pricing}, response, NOW, NOW) + await asyncio.shield(initial.receipt) + accepted: Final = database.rows + replay: Final = runtime.submit_batch(kwargs, response, NOW, NOW) + await asyncio.shield(replay.receipt) + assert database.rows == accepted and len(accepted) == 2 + assert runtime.meter is not None and runtime.meter.pending_bytes == 0 + finally: + await runtime.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py index f395c3b687d..bd3a0b9ed70 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_meter.py +++ b/tests/unit/proxy/spend_tracking/test_usage_meter.py @@ -78,6 +78,13 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N meter: Final = DurableUsageMeter( database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush ) + reservation: Final = meter.reserve_producer(size // 2, size // 2) + with pytest.raises(UsageCaptureError, match="producer byte capacity"): + meter.reserve_producer(1, size) + with pytest.raises(UsageCaptureError, match="producer byte capacity"): + meter.resize_producer(reservation, size, 1) + assert meter.pending_bytes == size // 2 + meter.release_producer(reservation) receipt: Final = meter.submit((write,)) with pytest.raises(UsageCaptureError, match="capacity"): meter.submit((_write("extra"),)) @@ -88,6 +95,8 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N assert database.committed == (write.event.event_id,) with pytest.raises(UsageCaptureError, match="closed"): meter.submit((_write("after-close"),)) + with pytest.raises(UsageCaptureError, match="producer byte capacity"): + meter.reserve_producer(1, 1) @pytest.mark.asyncio