mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(accounting): share byte admission across batch producers and writes
This commit is contained in:
parent
e517e65bc2
commit
a75be2bf78
4 changed files with 180 additions and 16 deletions
|
|
@ -2,7 +2,8 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
|
|
@ -623,7 +624,7 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int)
|
|||
class _BatchCapture:
|
||||
payload: SpendLogsPayload
|
||||
root: UsageWrite
|
||||
rows: tuple[Mapping[str, JsonValue], ...]
|
||||
rows: bytes
|
||||
prerequisites: tuple[asyncio.Future[None], ...]
|
||||
|
||||
|
||||
|
|
@ -641,25 +642,69 @@ def _prepare_batch_capture(
|
|||
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
|
||||
raise UsageCaptureError("Batch rows require one terminal job observation")
|
||||
return _BatchCapture(
|
||||
payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs))
|
||||
payload,
|
||||
_batch_root(builder.writes[0], rows),
|
||||
b"\n".join(_json(row).encode("utf-8") for row in rows),
|
||||
tuple(prior.receipt for prior in _prior_captures(kwargs)),
|
||||
)
|
||||
|
||||
|
||||
def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]:
|
||||
return (
|
||||
_batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index)
|
||||
for index, match in enumerate(re.finditer(rb"[^\n]+", rows))
|
||||
)
|
||||
|
||||
|
||||
def _batch_write_bound(root: UsageWrite, rows: bytes) -> int:
|
||||
return max(
|
||||
len(root.envelope.encode("utf-8")),
|
||||
max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)),
|
||||
)
|
||||
|
||||
|
||||
def _batch_capacity(
|
||||
meter: DurableUsageMeter, root: UsageWrite, rows: bytes, *, extra_bytes: int = 0
|
||||
) -> tuple[int, int, int]:
|
||||
retained: Final = len(rows) + len(root.envelope.encode("utf-8")) + extra_bytes
|
||||
bound: Final = _batch_write_bound(root, rows)
|
||||
chunk_bytes: Final = max(
|
||||
bound,
|
||||
min(
|
||||
bound * min(meter.settings.batch_size, rows.count(b"\n") + 1),
|
||||
meter.settings.max_pending_bytes // 4,
|
||||
(meter.settings.max_pending_bytes - retained) // 2,
|
||||
),
|
||||
)
|
||||
return retained + chunk_bytes, chunk_bytes, bound
|
||||
|
||||
|
||||
async def _persist_batch_capture(
|
||||
meter: DurableUsageMeter,
|
||||
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]],
|
||||
capture: _BatchCapture,
|
||||
root: UsageWrite,
|
||||
rows: bytes,
|
||||
prerequisites: tuple[asyncio.Future[None], ...],
|
||||
chunk_bytes: int,
|
||||
reservation: object | None = None,
|
||||
) -> None:
|
||||
for receipt in capture.prerequisites:
|
||||
for receipt in prerequisites:
|
||||
await asyncio.shield(receipt)
|
||||
await meter.record_writes((capture.root,))
|
||||
accepted: Final = await load((capture.root.event.event_id,))
|
||||
await meter.record_writes((root,))
|
||||
accepted: Final = await load((root.event.event_id,))
|
||||
if len(accepted) != 1:
|
||||
raise UsageCaptureError("Committed batch snapshot could not be loaded")
|
||||
frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0])
|
||||
remaining: Final = iter(enumerate(capture.rows))
|
||||
while group := tuple(islice(remaining, meter.settings.batch_size)):
|
||||
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
|
||||
frozen: Final = _reuse_terminal_snapshot(root, accepted[0])
|
||||
retained, capacity, bound = _batch_capacity(meter, frozen, rows, extra_bytes=len(root.envelope.encode("utf-8")))
|
||||
if reservation is not None:
|
||||
meter.resize_producer(reservation, retained, capacity)
|
||||
write_capacity: Final = capacity if reservation is not None else chunk_bytes
|
||||
if bound > write_capacity:
|
||||
raise UsageCaptureError("Batch observation exceeds reserved producer write capacity")
|
||||
group_size: Final = min(meter.settings.batch_size, max(1, write_capacity // bound))
|
||||
remaining: Final = _batch_writes(frozen, rows)
|
||||
while group := tuple(islice(remaining, group_size)):
|
||||
await meter.record_writes(group)
|
||||
|
||||
|
||||
async def capture_batch_callback(
|
||||
|
|
@ -673,7 +718,9 @@ async def capture_batch_callback(
|
|||
if load is None:
|
||||
raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader")
|
||||
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
|
||||
await _persist_batch_capture(meter, load, capture)
|
||||
await _persist_batch_capture(
|
||||
meter, load, capture.root, capture.rows, capture.prerequisites, meter.settings.max_pending_bytes
|
||||
)
|
||||
receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
|
||||
receipt.set_result(None)
|
||||
return CapturedUsage(capture.payload, (capture.root,), receipt)
|
||||
|
|
@ -986,7 +1033,15 @@ class UsageCaptureRuntime:
|
|||
if len(self._batches) >= meter.settings.max_pending_records:
|
||||
raise UsageCaptureError("Batch capture producer capacity exhausted")
|
||||
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
|
||||
task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture")
|
||||
retained, chunk_bytes, _ = _batch_capacity(meter, capture.root, capture.rows)
|
||||
reservation: Final = meter.reserve_producer(retained, chunk_bytes)
|
||||
task: Final = asyncio.create_task(
|
||||
_persist_batch_capture(
|
||||
meter, load, capture.root, capture.rows, capture.prerequisites, chunk_bytes, reservation
|
||||
),
|
||||
name="accounting batch capture",
|
||||
)
|
||||
task.add_done_callback(lambda _: meter.release_producer(reservation))
|
||||
self._batches.add(task)
|
||||
task.add_done_callback(self._batch_finished)
|
||||
return CapturedUsage(capture.payload, (capture.root,), task)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
from collections import deque
|
||||
from collections.abc import Awaitable, Callable
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from concurrent.futures import Future
|
||||
from dataclasses import dataclass
|
||||
from itertools import accumulate, chain, count, takewhile
|
||||
|
|
@ -115,6 +115,8 @@ class DurableUsageMeter:
|
|||
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
|
||||
self._wakeup: Final = asyncio.Event()
|
||||
self._capacity_changed: Final = asyncio.Event()
|
||||
self._reservations: Mapping[object, tuple[int, int]] = {}
|
||||
self._reserved_bytes = 0
|
||||
self._task: asyncio.Task[None] | None = None
|
||||
self._closed = False
|
||||
self.pending_bytes = 0
|
||||
|
|
@ -144,6 +146,36 @@ class DurableUsageMeter:
|
|||
wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
|
||||
return wrapped
|
||||
|
||||
def reserve_producer(self, size: int, headroom: int) -> object:
|
||||
token: Final = object()
|
||||
self.resize_producer(token, size, headroom)
|
||||
return token
|
||||
|
||||
def resize_producer(self, token: object, size: int, headroom: int) -> None:
|
||||
with self._admission:
|
||||
previous: Final = self._reservations.get(token, (0, 0))[0]
|
||||
delta: Final = size - previous
|
||||
others: Final = (value[1] for key, value in self._reservations.items() if key is not token)
|
||||
required: Final = max((headroom, *others))
|
||||
if (
|
||||
self._closed
|
||||
or self.pending_bytes + delta > self.settings.max_pending_bytes
|
||||
or self._reserved_bytes + delta + required > self.settings.max_pending_bytes
|
||||
):
|
||||
raise UsageCaptureError("Batch capture producer byte capacity exhausted")
|
||||
self._reservations = {**self._reservations, token: (size, headroom)}
|
||||
self._reserved_bytes += delta
|
||||
self.pending_bytes += delta
|
||||
self._capacity_changed.set()
|
||||
|
||||
def release_producer(self, token: object) -> None:
|
||||
with self._admission:
|
||||
size, _ = self._reservations[token]
|
||||
self._reservations = {key: value for key, value in self._reservations.items() if key is not token}
|
||||
self._reserved_bytes -= size
|
||||
self.pending_bytes -= size
|
||||
self._capacity_changed.set()
|
||||
|
||||
def _ensure_worker(self) -> None:
|
||||
if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
|
||||
self._wakeup.set()
|
||||
|
|
@ -156,14 +188,15 @@ class DurableUsageMeter:
|
|||
|
||||
async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None:
|
||||
size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
|
||||
capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes
|
||||
if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or (
|
||||
size > self.settings.max_pending_bytes and len(writes) > 1
|
||||
size > capacity and len(writes) > 1
|
||||
):
|
||||
middle: Final = len(writes) // 2
|
||||
await self.record_writes(writes[:middle])
|
||||
await self.record_writes(writes[middle:])
|
||||
return
|
||||
if size > self.settings.max_pending_bytes:
|
||||
if size > capacity:
|
||||
raise UsageCaptureError("One usage observation exceeds capture capacity")
|
||||
while (receipt := self._submit_available(writes)) is None:
|
||||
self._wakeup.set()
|
||||
|
|
|
|||
|
|
@ -1607,3 +1607,70 @@ async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence(
|
|||
release.set()
|
||||
await asyncio.gather(callback, return_exceptions=True)
|
||||
await runtime.stop(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() -> None:
|
||||
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
|
||||
|
||||
database: Final = StoredUsage()
|
||||
release: Final = asyncio.Event()
|
||||
runtime: Final = UsageCaptureRuntime()
|
||||
|
||||
async def persist(writes: tuple[UsageWrite, ...]) -> None:
|
||||
await release.wait()
|
||||
assert runtime.meter is not None
|
||||
assert runtime.meter.pending_bytes <= 65536
|
||||
await database.insert(writes)
|
||||
|
||||
runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load)
|
||||
rows: Final = tuple(
|
||||
{"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None}
|
||||
for index in range(1000)
|
||||
)
|
||||
response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}}
|
||||
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
|
||||
try:
|
||||
first: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
|
||||
assert runtime.meter is not None
|
||||
assert 49000 < runtime.meter.pending_bytes <= 65536
|
||||
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
|
||||
runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
|
||||
release.set()
|
||||
await asyncio.shield(first.receipt)
|
||||
assert len(database.rows) == 1001
|
||||
assert runtime.meter.pending_bytes == 0
|
||||
second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
|
||||
await asyncio.shield(second.receipt)
|
||||
assert len(database.rows) == 2002
|
||||
assert runtime.meter.pending_bytes == 0
|
||||
finally:
|
||||
release.set()
|
||||
await runtime.stop(1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_replay_reserves_the_accepted_pricing_snapshot_size() -> None:
|
||||
from litellm.proxy.spend_tracking.usage_capture import PricingCapture, UsageBatchWriter, UsageCaptureRuntime
|
||||
|
||||
database: Final = StoredUsage()
|
||||
writer: Final = UsageBatchWriter(database.insert, database.load)
|
||||
runtime: Final = UsageCaptureRuntime()
|
||||
runtime.start(writer.write, UsageCaptureSettings(max_pending_bytes=16384), load=database.load)
|
||||
response: Final = {
|
||||
"id": "batch",
|
||||
"status": "completed",
|
||||
"usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},)},
|
||||
}
|
||||
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
|
||||
pricing: Final = PricingCapture("{}", "{}", json.dumps({"batch_models": {"model": {"source": "x" * 1200}}}))
|
||||
try:
|
||||
initial: Final = runtime.submit_batch({**kwargs, "_accounting_pricing_snapshot": pricing}, response, NOW, NOW)
|
||||
await asyncio.shield(initial.receipt)
|
||||
accepted: Final = database.rows
|
||||
replay: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
|
||||
await asyncio.shield(replay.receipt)
|
||||
assert database.rows == accepted and len(accepted) == 2
|
||||
assert runtime.meter is not None and runtime.meter.pending_bytes == 0
|
||||
finally:
|
||||
await runtime.stop(1)
|
||||
|
|
|
|||
|
|
@ -78,6 +78,13 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N
|
|||
meter: Final = DurableUsageMeter(
|
||||
database.write, UsageCaptureSettings(max_pending_bytes=size), wait_for_batch=_manual_flush
|
||||
)
|
||||
reservation: Final = meter.reserve_producer(size // 2, size // 2)
|
||||
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
|
||||
meter.reserve_producer(1, size)
|
||||
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
|
||||
meter.resize_producer(reservation, size, 1)
|
||||
assert meter.pending_bytes == size // 2
|
||||
meter.release_producer(reservation)
|
||||
receipt: Final = meter.submit((write,))
|
||||
with pytest.raises(UsageCaptureError, match="capacity"):
|
||||
meter.submit((_write("extra"),))
|
||||
|
|
@ -88,6 +95,8 @@ async def test_byte_limit_rejects_new_work_without_evicting_admitted_work() -> N
|
|||
assert database.committed == (write.event.event_id,)
|
||||
with pytest.raises(UsageCaptureError, match="closed"):
|
||||
meter.submit((_write("after-close"),))
|
||||
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
|
||||
meter.reserve_producer(1, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue