fix(accounting): isolate batch persistence and preserve trusted attribution

This commit is contained in:
Joshua Valluru 2026-10-31 02:01:48 -07:00
parent 80f4a31bc2
commit e517e65bc2
9 changed files with 247 additions and 45 deletions

View file

@ -275,7 +275,7 @@ class AccountingLedger:
)
return tuple(
UsageWrite(
OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])),
_json(row["payload"]),
TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
)

View file

@ -630,7 +630,10 @@ class _ProxyDBLogger(CustomLogger):
if usage_capture.meter is None:
return None
captured_identity: Final = str(request_data.get("call_type")) in _CAPTURED_IDENTITY_CALL_TYPES
captured_identity: Final = (
_batch_accounting_for(request_data.get("litellm_call_id")) is not None
and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP
)
current_identity: Final = {
**LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
"user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),

View file

@ -619,6 +619,49 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int)
)
@dataclass(frozen=True, slots=True)
class _BatchCapture:
payload: SpendLogsPayload
root: UsageWrite
rows: tuple[Mapping[str, JsonValue], ...]
prerequisites: tuple[asyncio.Future[None], ...]
def _prepare_batch_capture(
kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime
) -> _BatchCapture:
from litellm.proxy.db.db_spend_update_writer import get_llm_router
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
rows: Final = tuple(sorted(_batch_rows(response), key=_json))
builder: Final = _CaptureBuilder(kwargs, response)
payload: Final = get_logging_payload(
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
)
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
raise UsageCaptureError("Batch rows require one terminal job observation")
return _BatchCapture(
payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs))
)
async def _persist_batch_capture(
meter: DurableUsageMeter,
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]],
capture: _BatchCapture,
) -> None:
for receipt in capture.prerequisites:
await asyncio.shield(receipt)
await meter.record_writes((capture.root,))
accepted: Final = await load((capture.root.event.event_id,))
if len(accepted) != 1:
raise UsageCaptureError("Committed batch snapshot could not be loaded")
frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0])
remaining: Final = iter(enumerate(capture.rows))
while group := tuple(islice(remaining, meter.settings.batch_size)):
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
async def capture_batch_callback(
meter: DurableUsageMeter,
load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None,
@ -627,32 +670,13 @@ async def capture_batch_callback(
start_time: datetime,
end_time: datetime,
) -> CapturedUsage:
from litellm.proxy.db.db_spend_update_writer import get_llm_router
from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
if load is None:
raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader")
rows: Final = tuple(sorted(_batch_rows(response), key=_json))
builder: Final = _CaptureBuilder(kwargs, response)
payload: Final = get_logging_payload(
dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe
)
if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None:
raise UsageCaptureError("Batch rows require one terminal job observation")
root: Final = _batch_root(builder.writes[0], rows)
for prior in _prior_captures(kwargs):
await asyncio.shield(prior.receipt)
await meter.record_writes((root,))
accepted: Final = await load((root.event.event_id,))
if len(accepted) != 1:
raise UsageCaptureError("Committed batch snapshot could not be loaded")
frozen: Final = _reuse_terminal_snapshot(root, accepted[0])
remaining: Final = iter(enumerate(rows))
while group := tuple(islice(remaining, meter.settings.batch_size)):
await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group))
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
await _persist_batch_capture(meter, load, capture)
receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future()
receipt.set_result(None)
return CapturedUsage(payload, (frozen,), receipt)
return CapturedUsage(capture.payload, (capture.root,), receipt)
async def captured_callback_kwargs(
@ -669,7 +693,7 @@ async def captured_callback_kwargs(
return kwargs
try:
captured: Final = (
await capture_batch_callback(usage_meter, usage_capture.load, kwargs, response, start_time, end_time)
usage_capture.submit_batch(kwargs, response, start_time, end_time)
if _batch_rows(response)
else capture_callback(usage_meter, kwargs, response, start_time, end_time)
)
@ -926,6 +950,8 @@ class UsageBatchWriter:
class UsageCaptureRuntime:
def __init__(self) -> None:
self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle
self._stopping = False
self.meter: DurableUsageMeter | None = None
self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None
@ -945,18 +971,51 @@ class UsageCaptureRuntime:
meter: Final = DurableUsageMeter(write_batch, settings)
self.meter = meter
self.load = load
self._stopping = False
Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage(
meter, logger, response, start, end, freeze
)
def submit_batch(
self, kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime
) -> CapturedUsage:
meter: Final = self.meter
load: Final = self.load
if meter is None or load is None or self._stopping:
raise UsageCaptureError("Batch capture requires an active ledger writer and snapshot reader")
if len(self._batches) >= meter.settings.max_pending_records:
raise UsageCaptureError("Batch capture producer capacity exhausted")
capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time)
task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture")
self._batches.add(task)
task.add_done_callback(self._batch_finished)
return CapturedUsage(capture.payload, (capture.root,), task)
def _batch_finished(self, task: asyncio.Task[None]) -> None:
self._batches.discard(task)
if task.cancelled():
verbose_proxy_logger.error("Batch usage capture stopped before complete persistence")
elif (error := task.exception()) is not None:
verbose_proxy_logger.error("Batch usage capture failed", exc_info=error)
async def stop(self, timeout: float) -> None:
from litellm.litellm_core_utils.litellm_logging import Logging
if self.meter is None:
return
self._stopping = True
deadline: Final = asyncio.get_running_loop().time() + timeout
try:
await self.meter.stop(timeout)
if self._batches:
await asyncio.wait(tuple(self._batches), timeout=max(0, timeout))
finally:
Logging.usage_capture_callback = None
self.meter = None
self.load = None
remaining: Final = tuple(self._batches)
for task in remaining:
task.cancel()
await asyncio.gather(*remaining, return_exceptions=True)
try:
await self.meter.stop(max(0, deadline - asyncio.get_running_loop().time()))
finally:
Logging.usage_capture_callback = None
self.meter = None
self.load = None

View file

@ -114,6 +114,7 @@ class DurableUsageMeter:
self._admission: Final = Lock()
self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue
self._wakeup: Final = asyncio.Event()
self._capacity_changed: Final = asyncio.Event()
self._task: asyncio.Task[None] | None = None
self._closed = False
self.pending_bytes = 0
@ -165,11 +166,13 @@ class DurableUsageMeter:
if size > self.settings.max_pending_bytes:
raise UsageCaptureError("One usage observation exceeds capture capacity")
while (receipt := self._submit_available(writes)) is None:
await self.flush()
self._wakeup.set()
await self._capacity_changed.wait()
self._wakeup.set()
await asyncio.shield(receipt)
def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None:
self._capacity_changed.clear()
try:
receipt: Final = self.submit(writes)
except UsageCaptureError:
@ -265,6 +268,7 @@ class DurableUsageMeter:
with self._admission:
self.pending_bytes -= item.size
self.pending_records -= len(item.writes)
self._capacity_changed.set()
if item.committed.done():
continue
if error is None:

View file

@ -540,7 +540,12 @@ async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
writer: _Writer, ledger_database: str, failure: str
) -> None:
client, store = writer
events: Final = (_usage("caller", inbound=True), _usage("attempt"))
events: Final = (
_usage("caller", inbound=True),
_usage("attempt").model_copy(
update={"metrics": (Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),)}
),
)
writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events)
@asynccontextmanager

View file

@ -116,3 +116,34 @@ async def test_conflicting_usage_within_one_batch_is_rejected_before_database_ac
changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}}))
with pytest.raises(AccountingConflictError, match="conflicting copies"):
await AccountingLedger(_no_transaction).insert_usage_batch((original, changed))
@pytest.mark.asyncio
async def test_loaded_usage_preserves_decimal_metrics_from_jsonb(event: InboundUsageEvent) -> None:
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import cast
from unittest.mock import AsyncMock
from prisma import Prisma
from litellm.accounting import OutboundUsageEvent
outbound: Final = OutboundUsageEvent(
**event.model_dump(exclude={"metrics"}),
attempt_id="attempt",
retry_of_attempt_id=None,
metrics=(Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),),
)
payload: Final = {"event": outbound.model_dump(mode="json"), "usage": {"prompt_tokens": 120000}}
database: Final = AsyncMock(spec=Prisma)
database.query_raw.return_value = [{"payload": payload, "targets": []}]
@asynccontextmanager
async def transaction() -> AsyncIterator[Prisma]:
yield cast(Prisma, database)
recovered: Final = await AccountingLedger(transaction).load_usage_batch((outbound.event_id,))
assert len(recovered) == 1
assert recovered[0].event == outbound
assert json.loads(recovered[0].envelope) == payload

View file

@ -4001,6 +4001,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
from litellm.accounting import DeliveryTarget, DimensionName
from litellm.proxy import proxy_server
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting
from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime
from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage
@ -4025,14 +4026,15 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
runtime: Final = UsageCaptureRuntime()
runtime.start(database.insert, UsageCaptureSettings())
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
owner: Final = "original-owner" if deferred else "owner"
rejected: Final = failure and deferred and not complete
owner: Final = "original-owner" if deferred and not rejected else "owner"
metadata: Final = {
"user_api_key": "a" * 64,
"user_api_key": "b" * 64 if rejected else "a" * 64,
**(
{
"user_api_key_user_id": owner,
"user_api_key_team_id": "team",
"user_api_key_org_id": "org",
"user_api_key_user_id": "forged-owner" if rejected else owner,
"user_api_key_team_id": "forged-team" if rejected else "team",
"user_api_key_org_id": "forged-org" if rejected else "org",
"user_api_key_project_id": None if deferred and not complete else "project",
"user_api_key_alias": "key",
"user_api_key_team_alias": "team-alias",
@ -4049,21 +4051,28 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
"start_time": _CALL_START,
"litellm_params": {"metadata": metadata},
}
if rejected:
from litellm.proxy.utils import REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY
kwargs[REJECTED_BEFORE_REQUEST_SETUP_KEY] = REJECTED_BEFORE_REQUEST_SETUP
logger: Final = ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer))
try:
if failure:
await logger.async_post_call_failure_hook(
kwargs,
HTTPException(401, "model not allowed"),
key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"),
)
else:
await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START)
with track_batch_accounting("attributed" if deferred and not rejected else "other"):
if failure:
await logger.async_post_call_failure_hook(
kwargs,
HTTPException(401, "model not allowed"),
key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"),
)
else:
await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START)
finally:
await runtime.stop(1)
assert len(writer.rows) == 1
assert len(database.rows) == 2
dimensions: Final = {item.name: item.value for item in database.rows[-1].event.dimensions}
if rejected:
assert dimensions.get(DimensionName.KEY) == key.api_key
assert {
name: dimensions.get(name)
for name in (DimensionName.USER, DimensionName.TEAM, DimensionName.ORGANIZATION, DimensionName.PROJECT)
@ -4071,7 +4080,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution(
DimensionName.USER: owner,
DimensionName.TEAM: "team",
DimensionName.ORGANIZATION: "org",
DimensionName.PROJECT: None if deferred and not complete else "project",
DimensionName.PROJECT: None if deferred and not complete and not rejected else "project",
}
assert set(database.rows[-1].targets) >= {DeliveryTarget.DAILY_TEAM_SPEND, DeliveryTarget.DAILY_ORGANIZATION_SPEND}
legacy: Final = TypeAdapter(dict[str, object]).validate_python(writer.rows[0]["kwargs"])

View file

@ -1551,3 +1551,59 @@ async def test_large_batch_recovers_partial_commit_with_frozen_pricing_and_rejec
assert database.rows == before_conflict
assert transactions.qsize() > 2
assert meter.pending_bytes == meter.pending_records == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("outcome", ["drain", "cancel", "failure"])
async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence(
outcome: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
from litellm.proxy import proxy_server
from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, UsageCaptureRuntime, captured_callback_kwargs
database: Final = StoredUsage()
entered: Final = asyncio.Event()
release: Final = asyncio.Event()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
entered.set()
await release.wait()
if outcome == "failure":
raise RuntimeError("Persistence unavailable")
await database.insert(writes)
runtime: Final = UsageCaptureRuntime()
runtime.start(persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), load=database.load)
monkeypatch.setattr(proxy_server, "usage_capture", runtime)
kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}}
response: Final = {
"id": "batch",
"status": "completed",
"usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},) * 3},
}
callback: Final = asyncio.create_task(captured_callback_kwargs(kwargs, response, NOW, NOW))
try:
await entered.wait()
assert callback.done(), "Batch persistence must not consume the legacy callback deadline"
captured: Final = (await callback)["_accounting_capture"]
assert isinstance(captured, CapturedUsage) and not captured.receipt.done()
second: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW)
assert isinstance(second.get("_accounting_capture"), CapturedUsage)
overflow: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW)
assert "_accounting_capture" not in overflow
assert "Batch capture producer capacity exhausted" in caplog.text
if outcome != "cancel":
release.set()
await runtime.stop(0 if outcome == "cancel" else 1)
assert captured.receipt.done()
assert captured.receipt.cancelled() is (outcome == "cancel")
if outcome == "failure":
assert isinstance(captured.receipt.exception(), RuntimeError)
assert "Batch usage capture failed" in caplog.text
assert len(database.rows) == (4 if outcome == "drain" else 0)
with pytest.raises(UsageCaptureError, match="active ledger writer"):
runtime.submit_batch(kwargs, response, NOW, NOW)
finally:
release.set()
await asyncio.gather(callback, return_exceptions=True)
await runtime.stop(1)

View file

@ -427,3 +427,38 @@ async def test_cancelling_bounded_capture_preserves_admitted_writes_without_admi
await meter.close()
assert database.committed == ("0", "1")
assert meter.pending_bytes == meter.pending_records == 0
@pytest.mark.asyncio
async def test_capacity_wait_retries_admission_before_other_producers_finish() -> None:
entered: Final = asyncio.Queue[tuple[str, ...]]()
release: Final = asyncio.Queue[None]()
async def persist(writes: tuple[UsageWrite, ...]) -> None:
entered.put_nowait(tuple(write.event.event_id for write in writes))
await release.get()
meter: Final = DurableUsageMeter(
persist, UsageCaptureSettings(batch_size=2, max_pending_records=3), wait_for_batch=_manual_flush
)
initial: Final = meter.submit((_write("first"), _write("second")))
remaining: Final = meter.submit((_write("remaining"),))
assert await entered.get() == ("first", "second")
recording: Final = asyncio.create_task(meter.record_writes((_write("target"),)))
try:
await asyncio.sleep(0)
release.put_nowait(None)
assert await entered.get() == ("remaining",)
await asyncio.sleep(0)
assert meter.pending_records == 2
release.put_nowait(None)
assert await entered.get() == ("target",)
release.put_nowait(None)
await recording
await asyncio.gather(initial, remaining)
finally:
recording.cancel()
await asyncio.gather(recording, return_exceptions=True)
release.put_nowait(None)
release.put_nowait(None)
await meter.close()