mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(accounting): compact repeated batch observations without losing requests
This commit is contained in:
parent
03701c207b
commit
cc86849b8a
2 changed files with 50 additions and 25 deletions
|
|
@ -8,7 +8,7 @@ from dataclasses import dataclass, replace
|
|||
from datetime import datetime, timezone
|
||||
from decimal import Decimal
|
||||
from hashlib import sha256
|
||||
from itertools import chain, islice
|
||||
from itertools import chain, groupby, islice, repeat
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from pydantic import BaseModel, JsonValue, TypeAdapter
|
||||
|
|
@ -644,22 +644,28 @@ def _prepare_batch_capture(
|
|||
return _BatchCapture(
|
||||
payload,
|
||||
_batch_root(builder.writes[0], rows),
|
||||
b"\n".join(_json(row).encode("utf-8") for row in rows),
|
||||
b"\n".join(_json((row, sum(1 for _ in group))).encode("utf-8") for row, group in groupby(map(_json, rows))),
|
||||
tuple(prior.receipt for prior in _prior_captures(kwargs)),
|
||||
)
|
||||
|
||||
|
||||
def _batch_groups(rows: bytes) -> Iterator[tuple[str, int]]:
|
||||
return (TypeAdapter(tuple[str, int]).validate_json(match.group()) for match in re.finditer(rb"[^\n]+", rows))
|
||||
|
||||
|
||||
def _batch_writes(root: UsageWrite, rows: bytes) -> Iterator[UsageWrite]:
|
||||
return (
|
||||
_batch_row_write(root, _JSON_OBJECT.validate_json(match.group()), index)
|
||||
for index, match in enumerate(re.finditer(rb"[^\n]+", rows))
|
||||
)
|
||||
expanded: Final = chain.from_iterable(repeat(row, count) for row, count in _batch_groups(rows))
|
||||
return (_batch_row_write(root, _JSON_OBJECT.validate_json(row), index) for index, row in enumerate(expanded))
|
||||
|
||||
|
||||
def _batch_write_bound(root: UsageWrite, rows: bytes) -> int:
|
||||
last_index: Final = sum(count for _, count in _batch_groups(rows)) - 1
|
||||
return max(
|
||||
len(root.envelope.encode("utf-8")),
|
||||
max(len(write.envelope.encode("utf-8")) for write in _batch_writes(root, rows)),
|
||||
max(
|
||||
len(_batch_row_write(root, _JSON_OBJECT.validate_json(row), last_index).envelope.encode("utf-8"))
|
||||
for row, _ in _batch_groups(rows)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -671,7 +677,7 @@ def _batch_capacity(
|
|||
chunk_bytes: Final = max(
|
||||
bound,
|
||||
min(
|
||||
bound * min(meter.settings.batch_size, rows.count(b"\n") + 1),
|
||||
bound * min(meter.settings.batch_size, sum(count for _, count in _batch_groups(rows))),
|
||||
meter.settings.max_pending_bytes // 4,
|
||||
(meter.settings.max_pending_bytes - retained) // 2,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -1624,32 +1624,51 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain()
|
|||
await database.insert(writes)
|
||||
|
||||
runtime.start(persist, UsageCaptureSettings(batch_size=32, max_pending_bytes=65536), load=database.load)
|
||||
rows: Final = tuple(
|
||||
{"model": "capture-model" if index % 7 else None, "usage": {"prompt_tokens": 2} if index % 7 else None}
|
||||
for index in range(1000)
|
||||
from litellm.batches.batch_utils import batch_cost_and_usage
|
||||
|
||||
parsed: Final = batch_cost_and_usage(
|
||||
[
|
||||
{
|
||||
"response": {
|
||||
"status_code": 200,
|
||||
"body": {
|
||||
"model": "capture-model",
|
||||
"usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3},
|
||||
},
|
||||
}
|
||||
}
|
||||
for _ in range(1000)
|
||||
],
|
||||
"openai",
|
||||
model_info={},
|
||||
)
|
||||
response: Final = {"id": "batch", "status": "completed", "usage": {"batch_usage_by_request": rows}}
|
||||
rows: Final = ({"model": "capture-model", "usage": {"prompt_tokens": 2}},)
|
||||
response: Final = {"id": "batch", "status": "completed", "usage": parsed.usage}
|
||||
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
|
||||
try:
|
||||
first: Final = runtime.submit_batch(kwargs, response, NOW, NOW)
|
||||
assert runtime.meter is not None
|
||||
assert 49000 < runtime.meter.pending_bytes <= 65536
|
||||
with pytest.raises(UsageCaptureError, match="producer byte capacity"):
|
||||
runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
|
||||
large: Final = asyncio.Queue[asyncio.Future[None]]()
|
||||
for index in range(100):
|
||||
try:
|
||||
captured: Final = runtime.submit_batch(kwargs, {**response, "id": f"batch-{index}"}, NOW, NOW)
|
||||
except UsageCaptureError:
|
||||
break
|
||||
large.put_nowait(captured.receipt)
|
||||
large_count: Final = large.qsize()
|
||||
assert 0 < large_count < 100
|
||||
assert runtime.meter is not None and 0 < runtime.meter.pending_bytes <= 65536
|
||||
release.set()
|
||||
await asyncio.shield(first.receipt)
|
||||
assert len(database.rows) == 1001
|
||||
assert runtime.meter.pending_bytes == 0
|
||||
second: Final = runtime.submit_batch(kwargs, {**response, "id": "other-batch"}, NOW, NOW)
|
||||
await asyncio.shield(second.receipt)
|
||||
assert len(database.rows) == 2002
|
||||
await asyncio.gather(*(large.get_nowait() for _ in range(large_count)))
|
||||
assert len(database.rows) == 1001 * large_count
|
||||
observations: Final = tuple(json.loads(write.envelope) for write in database.rows)
|
||||
assert sum(row["usage"].get("prompt_tokens", 0) for row in observations) == 2000 * large_count
|
||||
assert sum(row["usage"].get("completion_tokens", 0) for row in observations) == 1000 * large_count
|
||||
assert runtime.meter.pending_bytes == 0
|
||||
admitted: Final = asyncio.Queue[asyncio.Future[None]]()
|
||||
for index in range(100):
|
||||
try:
|
||||
small: Final = runtime.submit_batch(
|
||||
kwargs,
|
||||
{**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows[1:2]}},
|
||||
{**response, "id": f"small-{index}", "usage": {"batch_usage_by_request": rows}},
|
||||
NOW,
|
||||
NOW,
|
||||
)
|
||||
|
|
@ -1659,7 +1678,7 @@ async def test_batch_producers_share_byte_capacity_and_release_it_after_drain()
|
|||
count: Final = admitted.qsize()
|
||||
assert 1 < count < 100
|
||||
await asyncio.gather(*(admitted.get_nowait() for _ in range(count)))
|
||||
assert len(database.rows) == 2002 + 2 * count
|
||||
assert len(database.rows) == 1001 * large_count + 2 * count
|
||||
assert runtime.meter.pending_bytes == 0
|
||||
finally:
|
||||
release.set()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue