diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index dc4bb3826de..19480a66678 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone from decimal import Decimal from hashlib import sha256 -from itertools import chain, islice +from itertools import chain, groupby, islice, repeat from typing import TYPE_CHECKING, Final from pydantic import BaseModel, JsonValue, TypeAdapter @@ -644,22 +644,28 @@ def _prepare_batch_capture( return _BatchCapture( payload, _batch_root(builder.writes[0], rows), - b"\n".join(_json(row).encode("utf-8") for row in rows), + b"\n".join(_json((row, sum(1 for _ in group))).encode("utf-8") for row, group in groupby(map(_json, rows))), tuple(prior.receipt for prior in _prior_captures(kwargs)), ) +def _batch_groups(rows: bytes) -> Iterator[tuple[str, int]]: + return (TypeAdapter(tuple[str, int]).validate_json(match.group()) for match in re.finditer(rb"[^\n]+", rows)) + + def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]: - return ( - _batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index) - for index, match in enumerate(re.finditer(rb"[^\n]+", rows)) - ) + expanded: Final = chain.from_iterable(repeat(row, count) for row, count in _batch_groups(rows)) + return (_batch_row_write(root, _JSON_OBJECT.validate_json(row), index) for index, row in enumerate(expanded)) def _batch_write_bound(root: UsageWrite, rows: bytes) -> int: + last_index: Final = sum(count for _, count in _batch_groups(rows)) - 1 return max( len(root.envelope.encode("utf-8")), - max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)), + max( + len(_batch_row_write(root, _JSON_OBJECT.validate_json(row), last_index).envelope.encode("utf-8")) + for row, _ in _batch_groups(rows) + ), ) @@ -671,7 +677,7 @@ def _batch_capacity( chunk_bytes: Final = max( bound, min( - bound * min(meter.settings.batch_size, rows.count(b"\n") + 1), + bound * min(meter.settings.batch_size, sum(count for _, count in _batch_groups(rows))), meter.settings.max_pending_bytes // 4, (meter.settings.max_pending_bytes - retained) // 2, ), diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 38374c138f5..32710e594ee 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1624,32 +1624,51 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() await database.insert(writes) runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load) - rows: Final = tuple( - {"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None} - for index in range(1000) + from litellm.batches.batch_utils import batch_cost_and_usage + + parsed: Final = batch_cost_and_usage( + [ + { + "response": { + "status_code": 200, + "body": { + "model": "capture-model", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + }, + } + } + for _ in range(1000) + ], + "openai", + model_info={}, ) - response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}} + rows: Final = ({"model": "capture-model", "usage": {"prompt_tokens": 2}},) + response: Final = {"id": "batch", "status": "completed", "usage": parsed.usage} kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} try: - first: Final = runtime.submit_batch(kwargs, response, NOW, NOW) - assert runtime.meter is not None - assert 49000 < runtime.meter.pending_bytes <= 65536 - with pytest.raises(UsageCaptureError, match="producer byte capacity"): - runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW) + large: Final = asyncio.Queue[asyncio.Future[None]]() + for index in range(100): + try: + captured: Final = runtime.submit_batch(kwargs, {**response, "id": f"batch-{index}"}, NOW, NOW) + except UsageCaptureError: + break + large.put_nowait(captured.receipt) + large_count: Final = large.qsize() + assert 0 < large_count < 100 + assert runtime.meter is not None and 0 < runtime.meter.pending_bytes <= 65536 release.set() - await asyncio.shield(first.receipt) - assert len(database.rows) == 1001 - assert runtime.meter.pending_bytes == 0 - second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW) - await asyncio.shield(second.receipt) - assert len(database.rows) == 2002 + await asyncio.gather(*(large.get_nowait() for _ in range(large_count))) + assert len(database.rows) == 1001 * large_count + observations: Final = tuple(json.loads(write.envelope) for write in database.rows) + assert sum(row["usage"].get("prompt_tokens", 0) for row in observations) == 2000 * large_count + assert sum(row["usage"].get("completion_tokens", 0) for row in observations) == 1000 * large_count assert runtime.meter.pending_bytes == 0 admitted: Final = asyncio.Queue[asyncio.Future[None]]() for index in range(100): try: small: Final = runtime.submit_batch( kwargs, - {**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows[1:2]}}, + {**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows}}, NOW, NOW, ) @@ -1659,7 +1678,7 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain() count: Final = admitted.qsize() assert 1 < count < 100 await asyncio.gather(*(admitted.get_nowait() for _ in range(count))) - assert len(database.rows) == 2002 + 2 * count + assert len(database.rows) == 1001 * large_count + 2 * count assert runtime.meter.pending_bytes == 0 finally: release.set()