fix(accounting): compact repeated batch observations without losing requests

This commit is contained in:
Joshua Valluru 2026-10-31 03:47:24 -07:00
parent 03701c207b
commit cc86849b8a
2 changed files with 50 additions and 25 deletions

View file

@ -8,7 +8,7 @@ from dataclasses import dataclass, replace
from datetime import datetime, timezone
from decimal import Decimal
from hashlib import sha256
from itertools import chain, islice
from itertools import chain, groupby, islice, repeat
from typing import TYPE_CHECKING, Final
from pydantic import BaseModel, JsonValue, TypeAdapter
@ -644,22 +644,28 @@ def _prepare_batch_capture(
return _BatchCapture(
payload,
_batch_root(builder.writes[0], rows),
b"\n".join(_json(row).encode("utf-8") for row in rows),
b"\n".join(_json((row, sum(1 for _ in group))).encode("utf-8") for row, group in groupby(map(_json, rows))),
tuple(prior.receipt for prior in _prior_captures(kwargs)),
)
def _batch_groups(rows: bytes) -> Iterator[tuple[str, int]]:
return (TypeAdapter(tuple[str, int]).validate_json(match.group()) for match in re.finditer(rb"[^\n]+", rows))
def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]:
return (
_batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index)
for index, match in enumerate(re.finditer(rb"[^\n]+", rows))
)
expanded: Final = chain.from_iterable(repeat(row, count) for row, count in _batch_groups(rows))
return (_batch_row_write(root, _JSON_OBJECT.validate_json(row), index) for index, row in enumerate(expanded))
def _batch_write_bound(root: UsageWrite, rows: bytes) -> int:
last_index: Final = sum(count for _, count in _batch_groups(rows)) - 1
return max(
len(root.envelope.encode("utf-8")),
max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)),
max(
len(_batch_row_write(root, _JSON_OBJECT.validate_json(row), last_index).envelope.encode("utf-8"))
for row, _ in _batch_groups(rows)
),
)
@ -671,7 +677,7 @@ def _batch_capacity(
chunk_bytes: Final = max(
bound,
min(
bound * min(meter.settings.batch_size, rows.count(b"\n") + 1),
bound * min(meter.settings.batch_size, sum(count for _, count in _batch_groups(rows))),
meter.settings.max_pending_bytes // 4,
(meter.settings.max_pending_bytes - retained) // 2,
),

View file

@ -1624,32 +1624,51 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain()
await database.insert(writes)
runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load)
rows: Final = tuple(
{"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None}
for index in range(1000)
from litellm.batches.batch_utils import batch_cost_and_usage
parsed: Final = batch_cost_and_usage(
[
{
"response": {
"status_code": 200,
"body": {
"model": "capture-model",
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
},
}
}
for _ in range(1000)
],
"openai",
model_info={},
)
response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}}
rows: Final = ({"model": "capture-model", "usage": {"prompt_tokens": 2}},)
response: Final = {"id": "batch", "status": "completed", "usage": parsed.usage}
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
try:
first: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
assert runtime.meter is not None
assert 49000 < runtime.meter.pending_bytes <= 65536
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
large: Final = asyncio.Queue[asyncio.Future[None]]()
for index in range(100):
try:
captured: Final = runtime.submit_batch(kwargs, {**response, "id": f"batch-{index}"}, NOW, NOW)
except UsageCaptureError:
break
large.put_nowait(captured.receipt)
large_count: Final = large.qsize()
assert 0 < large_count < 100
assert runtime.meter is not None and 0 < runtime.meter.pending_bytes <= 65536
release.set()
await asyncio.shield(first.receipt)
assert len(database.rows) == 1001
assert runtime.meter.pending_bytes == 0
second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
await asyncio.shield(second.receipt)
assert len(database.rows) == 2002
await asyncio.gather(*(large.get_nowait() for _ in range(large_count)))
assert len(database.rows) == 1001 * large_count
observations: Final = tuple(json.loads(write.envelope) for write in database.rows)
assert sum(row["usage"].get("prompt_tokens", 0) for row in observations) == 2000 * large_count
assert sum(row["usage"].get("completion_tokens", 0) for row in observations) == 1000 * large_count
assert runtime.meter.pending_bytes == 0
admitted: Final = asyncio.Queue[asyncio.Future[None]]()
for index in range(100):
try:
small: Final = runtime.submit_batch(
kwargs,
{**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows[1:2]}},
{**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows}},
NOW,
NOW,
)
@ -1659,7 +1678,7 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain()
count: Final = admitted.qsize()
assert 1 < count < 100
await asyncio.gather(*(admitted.get_nowait() for _ in range(count)))
assert len(database.rows) == 2002 + 2 * count
assert len(database.rows) == 1001 * large_count + 2 * count
assert runtime.meter.pending_bytes == 0
finally:
release.set()