From e517e65bc2c0eb46439e675e29b45f4d794a8dfe Mon Sep 17 00:00:00 2001 From: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:01:48 -0700 Subject: [PATCH] fix(accounting): isolate batch persistence and preserve trusted attribution --- litellm/proxy/db/accounting_ledger.py | 2 +- .../proxy/hooks/proxy_track_cost_callback.py | 5 +- litellm/proxy/spend_tracking/usage_capture.py | 113 +++++++++++++----- litellm/proxy/spend_tracking/usage_meter.py | 6 +- .../database/test_accounting_ledger.py | 7 +- tests/unit/proxy/db/test_accounting_ledger.py | 31 +++++ .../hooks/test_proxy_track_cost_callback.py | 37 +++--- .../spend_tracking/test_usage_capture.py | 56 +++++++++ .../proxy/spend_tracking/test_usage_meter.py | 35 ++++++ 9 files changed, 247 insertions(+), 45 deletions(-) diff --git a/litellm/proxy/db/accounting_ledger.py b/litellm/proxy/db/accounting_ledger.py index e37ce8d14d4..0a9c5db1619 100644 --- a/litellm/proxy/db/accounting_ledger.py +++ b/litellm/proxy/db/accounting_ledger.py @@ -275,7 +275,7 @@ class AccountingLedger: ) return tuple( UsageWrite( - OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]), + OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])), _json(row["payload"]), TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]), ) diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 1947693bd7c..980f0a5d410 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -630,7 +630,10 @@ class _ProxyDBLogger(CustomLogger): if usage_capture.meter is None: return None - captured_identity: Final = str(request_data.get("call_type")) in _CAPTURED_IDENTITY_CALL_TYPES + captured_identity: Final = ( + _batch_accounting_for(request_data.get("litellm_call_id")) is not None + and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP + ) current_identity: Final = { **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth), "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth), diff --git a/litellm/proxy/spend_tracking/usage_capture.py b/litellm/proxy/spend_tracking/usage_capture.py index 6b81f56b3af..a8e76505947 100644 --- a/litellm/proxy/spend_tracking/usage_capture.py +++ b/litellm/proxy/spend_tracking/usage_capture.py @@ -619,6 +619,49 @@ def _batch_row_write(root: UsageWrite, row: Mapping[str, JsonValue], index: int) ) +@dataclass(frozen=True, slots=True) +class _BatchCapture: + payload: SpendLogsPayload + root: UsageWrite + rows: tuple[Mapping[str, JsonValue], ...] + prerequisites: tuple[asyncio.Future[None], ...] + + +def _prepare_batch_capture( + kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime +) -> _BatchCapture: + from litellm.proxy.db.db_spend_update_writer import get_llm_router + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + + rows: Final = tuple(sorted(_batch_rows(response), key=_json)) + builder: Final = _CaptureBuilder(kwargs, response) + payload: Final = get_logging_payload( + dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe + ) + if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None: + raise UsageCaptureError("Batch rows require one terminal job observation") + return _BatchCapture( + payload, _batch_root(builder.writes[0], rows), rows, tuple(prior.receipt for prior in _prior_captures(kwargs)) + ) + + +async def _persist_batch_capture( + meter: DurableUsageMeter, + load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]], + capture: _BatchCapture, +) -> None: + for receipt in capture.prerequisites: + await asyncio.shield(receipt) + await meter.record_writes((capture.root,)) + accepted: Final = await load((capture.root.event.event_id,)) + if len(accepted) != 1: + raise UsageCaptureError("Committed batch snapshot could not be loaded") + frozen: Final = _reuse_terminal_snapshot(capture.root, accepted[0]) + remaining: Final = iter(enumerate(capture.rows)) + while group := tuple(islice(remaining, meter.settings.batch_size)): + await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group)) + + async def capture_batch_callback( meter: DurableUsageMeter, load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None, @@ -627,32 +670,13 @@ async def capture_batch_callback( start_time: datetime, end_time: datetime, ) -> CapturedUsage: - from litellm.proxy.db.db_spend_update_writer import get_llm_router - from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload - if load is None: raise UsageCaptureError("Batch capture requires the existing ledger snapshot reader") - rows: Final = tuple(sorted(_batch_rows(response), key=_json)) - builder: Final = _CaptureBuilder(kwargs, response) - payload: Final = get_logging_payload( - dict(kwargs), response, start_time, end_time, llm_router=get_llm_router(), capture_usage=builder.observe - ) - if len(builder.writes) != 1 or _terminal_facts(builder.writes[0]) is None: - raise UsageCaptureError("Batch rows require one terminal job observation") - root: Final = _batch_root(builder.writes[0], rows) - for prior in _prior_captures(kwargs): - await asyncio.shield(prior.receipt) - await meter.record_writes((root,)) - accepted: Final = await load((root.event.event_id,)) - if len(accepted) != 1: - raise UsageCaptureError("Committed batch snapshot could not be loaded") - frozen: Final = _reuse_terminal_snapshot(root, accepted[0]) - remaining: Final = iter(enumerate(rows)) - while group := tuple(islice(remaining, meter.settings.batch_size)): - await meter.record_writes(tuple(_batch_row_write(frozen, row, index) for index, row in group)) + capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) + await _persist_batch_capture(meter, load, capture) receipt: Final[asyncio.Future[None]] = asyncio.get_running_loop().create_future() receipt.set_result(None) - return CapturedUsage(payload, (frozen,), receipt) + return CapturedUsage(capture.payload, (capture.root,), receipt) async def captured_callback_kwargs( @@ -669,7 +693,7 @@ async def captured_callback_kwargs( return kwargs try: captured: Final = ( - await capture_batch_callback(usage_meter, usage_capture.load, kwargs, response, start_time, end_time) + usage_capture.submit_batch(kwargs, response, start_time, end_time) if _batch_rows(response) else capture_callback(usage_meter, kwargs, response, start_time, end_time) ) @@ -926,6 +950,8 @@ class UsageBatchWriter: class UsageCaptureRuntime: def __init__(self) -> None: + self._batches: Final[set[asyncio.Task[None]]] = set() # mutable-ok: owned producer lifecycle + self._stopping = False self.meter: DurableUsageMeter | None = None self.load: Callable[[tuple[str, ...]], Awaitable[tuple[UsageWrite, ...]]] | None = None @@ -945,18 +971,51 @@ class UsageCaptureRuntime: meter: Final = DurableUsageMeter(write_batch, settings) self.meter = meter self.load = load + self._stopping = False Logging.usage_capture_callback = lambda logger, response, start, end, freeze: capture_logging_usage( meter, logger, response, start, end, freeze ) + def submit_batch( + self, kwargs: Mapping[str, object], response: object, start_time: datetime, end_time: datetime + ) -> CapturedUsage: + meter: Final = self.meter + load: Final = self.load + if meter is None or load is None or self._stopping: + raise UsageCaptureError("Batch capture requires an active ledger writer and snapshot reader") + if len(self._batches) >= meter.settings.max_pending_records: + raise UsageCaptureError("Batch capture producer capacity exhausted") + capture: Final = _prepare_batch_capture(kwargs, response, start_time, end_time) + task: Final = asyncio.create_task(_persist_batch_capture(meter, load, capture), name="accounting batch capture") + self._batches.add(task) + task.add_done_callback(self._batch_finished) + return CapturedUsage(capture.payload, (capture.root,), task) + + def _batch_finished(self, task: asyncio.Task[None]) -> None: + self._batches.discard(task) + if task.cancelled(): + verbose_proxy_logger.error("Batch usage capture stopped before complete persistence") + elif (error := task.exception()) is not None: + verbose_proxy_logger.error("Batch usage capture failed", exc_info=error) + async def stop(self, timeout: float) -> None: from litellm.litellm_core_utils.litellm_logging import Logging if self.meter is None: return + self._stopping = True + deadline: Final = asyncio.get_running_loop().time() + timeout try: - await self.meter.stop(timeout) + if self._batches: + await asyncio.wait(tuple(self._batches), timeout=max(0, timeout)) finally: - Logging.usage_capture_callback = None - self.meter = None - self.load = None + remaining: Final = tuple(self._batches) + for task in remaining: + task.cancel() + await asyncio.gather(*remaining, return_exceptions=True) + try: + await self.meter.stop(max(0, deadline - asyncio.get_running_loop().time())) + finally: + Logging.usage_capture_callback = None + self.meter = None + self.load = None diff --git a/litellm/proxy/spend_tracking/usage_meter.py b/litellm/proxy/spend_tracking/usage_meter.py index 8608567da2a..c776facfff6 100644 --- a/litellm/proxy/spend_tracking/usage_meter.py +++ b/litellm/proxy/spend_tracking/usage_meter.py @@ -114,6 +114,7 @@ class DurableUsageMeter: self._admission: Final = Lock() self._queue: Final[deque[_Pending]] = deque() # mutable-ok: one bounded producer/consumer queue self._wakeup: Final = asyncio.Event() + self._capacity_changed: Final = asyncio.Event() self._task: asyncio.Task[None] | None = None self._closed = False self.pending_bytes = 0 @@ -165,11 +166,13 @@ class DurableUsageMeter: if size > self.settings.max_pending_bytes: raise UsageCaptureError("One usage observation exceeds capture capacity") while (receipt := self._submit_available(writes)) is None: - await self.flush() + self._wakeup.set() + await self._capacity_changed.wait() self._wakeup.set() await asyncio.shield(receipt) def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None: + self._capacity_changed.clear() try: receipt: Final = self.submit(writes) except UsageCaptureError: @@ -265,6 +268,7 @@ class DurableUsageMeter: with self._admission: self.pending_bytes -= item.size self.pending_records -= len(item.writes) + self._capacity_changed.set() if item.committed.done(): continue if error is None: diff --git a/tests/integration/database/test_accounting_ledger.py b/tests/integration/database/test_accounting_ledger.py index 86c2b325226..15cd5413149 100644 --- a/tests/integration/database/test_accounting_ledger.py +++ b/tests/integration/database/test_accounting_ledger.py @@ -540,7 +540,12 @@ async def test_usage_batch_is_atomic_and_replay_preserves_receipts( writer: _Writer, ledger_database: str, failure: str ) -> None: client, store = writer - events: Final = (_usage("caller", inbound=True), _usage("attempt")) + events: Final = ( + _usage("caller", inbound=True), + _usage("attempt").model_copy( + update={"metrics": (Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),)} + ), + ) writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events) @asynccontextmanager diff --git a/tests/unit/proxy/db/test_accounting_ledger.py b/tests/unit/proxy/db/test_accounting_ledger.py index 6ca719ed849..6b301146301 100644 --- a/tests/unit/proxy/db/test_accounting_ledger.py +++ b/tests/unit/proxy/db/test_accounting_ledger.py @@ -116,3 +116,34 @@ async def test_conflicting_usage_within_one_batch_is_rejected_before_database_ac changed: Final = UsageWrite(event, json.dumps({"event": event.model_dump(mode="json"), "pricing": {"rate": 2}})) with pytest.raises(AccountingConflictError, match="conflicting copies"): await AccountingLedger(_no_transaction).insert_usage_batch((original, changed)) + + +@pytest.mark.asyncio +async def test_loaded_usage_preserves_decimal_metrics_from_jsonb(event: InboundUsageEvent) -> None: + from collections.abc import AsyncIterator + from contextlib import asynccontextmanager + from typing import cast + from unittest.mock import AsyncMock + + from prisma import Prisma + + from litellm.accounting import OutboundUsageEvent + + outbound: Final = OutboundUsageEvent( + **event.model_dump(exclude={"metrics"}), + attempt_id="attempt", + retry_of_attempt_id=None, + metrics=(Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),), + ) + payload: Final = {"event": outbound.model_dump(mode="json"), "usage": {"prompt_tokens": 120000}} + database: Final = AsyncMock(spec=Prisma) + database.query_raw.return_value = [{"payload": payload, "targets": []}] + + @asynccontextmanager + async def transaction() -> AsyncIterator[Prisma]: + yield cast(Prisma, database) + + recovered: Final = await AccountingLedger(transaction).load_usage_batch((outbound.event_id,)) + assert len(recovered) == 1 + assert recovered[0].event == outbound + assert json.loads(recovered[0].envelope) == payload diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index adf95e3efb1..a4e78a48a1a 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -4001,6 +4001,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution( from litellm.accounting import DeliveryTarget, DimensionName from litellm.proxy import proxy_server from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.hooks.proxy_track_cost_callback import track_batch_accounting from litellm.proxy.spend_tracking.usage_capture import UsageCaptureRuntime from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings from tests.unit.proxy.spend_tracking.test_usage_capture import StoredUsage @@ -4025,14 +4026,15 @@ async def test_capture_and_legacy_share_resolved_key_attribution( runtime: Final = UsageCaptureRuntime() runtime.start(database.insert, UsageCaptureSettings()) monkeypatch.setattr(proxy_server, "usage_capture", runtime) - owner: Final = "original-owner" if deferred else "owner" + rejected: Final = failure and deferred and not complete + owner: Final = "original-owner" if deferred and not rejected else "owner" metadata: Final = { - "user_api_key": "a" * 64, + "user_api_key": "b" * 64 if rejected else "a" * 64, **( { - "user_api_key_user_id": owner, - "user_api_key_team_id": "team", - "user_api_key_org_id": "org", + "user_api_key_user_id": "forged-owner" if rejected else owner, + "user_api_key_team_id": "forged-team" if rejected else "team", + "user_api_key_org_id": "forged-org" if rejected else "org", "user_api_key_project_id": None if deferred and not complete else "project", "user_api_key_alias": "key", "user_api_key_team_alias": "team-alias", @@ -4049,21 +4051,28 @@ async def test_capture_and_legacy_share_resolved_key_attribution( "start_time": _CALL_START, "litellm_params": {"metadata": metadata}, } + if rejected: + from litellm.proxy.utils import REJECTED_BEFORE_REQUEST_SETUP, REJECTED_BEFORE_REQUEST_SETUP_KEY + + kwargs[REJECTED_BEFORE_REQUEST_SETUP_KEY] = REJECTED_BEFORE_REQUEST_SETUP logger: Final = ProxyDBLogger(spend_writer=lambda: cast(DBSpendUpdateWriter, writer)) try: - if failure: - await logger.async_post_call_failure_hook( - kwargs, - HTTPException(401, "model not allowed"), - key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"), - ) - else: - await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START) + with track_batch_accounting("attributed" if deferred and not rejected else "other"): + if failure: + await logger.async_post_call_failure_hook( + kwargs, + HTTPException(401, "model not allowed"), + key if complete else UserAPIKeyAuth(api_key="a" * 64, request_route="/v1/chat/completions"), + ) + else: + await logger.async_log_success_event(kwargs, {"id": "tool"}, _CALL_START, _CALL_START) finally: await runtime.stop(1) assert len(writer.rows) == 1 assert len(database.rows) == 2 dimensions: Final = {item.name: item.value for item in database.rows[-1].event.dimensions} + if rejected: + assert dimensions.get(DimensionName.KEY) == key.api_key assert { name: dimensions.get(name) for name in (DimensionName.USER, DimensionName.TEAM, DimensionName.ORGANIZATION, DimensionName.PROJECT) @@ -4071,7 +4080,7 @@ async def test_capture_and_legacy_share_resolved_key_attribution( DimensionName.USER: owner, DimensionName.TEAM: "team", DimensionName.ORGANIZATION: "org", - DimensionName.PROJECT: None if deferred and not complete else "project", + DimensionName.PROJECT: None if deferred and not complete and not rejected else "project", } assert set(database.rows[-1].targets) >= {DeliveryTarget.DAILY_TEAM_SPEND, DeliveryTarget.DAILY_ORGANIZATION_SPEND} legacy: Final = TypeAdapter(dict[str, object]).validate_python(writer.rows[0]["kwargs"]) diff --git a/tests/unit/proxy/spend_tracking/test_usage_capture.py b/tests/unit/proxy/spend_tracking/test_usage_capture.py index 51507fe8a1c..c89f1bd91d4 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_capture.py +++ b/tests/unit/proxy/spend_tracking/test_usage_capture.py @@ -1551,3 +1551,59 @@ async def test_large_batch_recovers_partial_commit_with_frozen_pricing_and_rejec assert database.rows == before_conflict assert transactions.qsize() > 2 assert meter.pending_bytes == meter.pending_records == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["drain", "cancel", "failure"]) +async def test_batch_callback_handoff_is_bounded_and_runtime_owns_persistence( + outcome: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, UsageCaptureRuntime, captured_callback_kwargs + + database: Final = StoredUsage() + entered: Final = asyncio.Event() + release: Final = asyncio.Event() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + entered.set() + await release.wait() + if outcome == "failure": + raise RuntimeError("Persistence unavailable") + await database.insert(writes) + + runtime: Final = UsageCaptureRuntime() + runtime.start(persist, UsageCaptureSettings(batch_size=2, max_pending_records=2), load=database.load) + monkeypatch.setattr(proxy_server, "usage_capture", runtime) + kwargs: Final = {"call_type": "aretrieve_batch", "litellm_call_id": "poll", "litellm_params": {}} + response: Final = { + "id": "batch", + "status": "completed", + "usage": {"batch_usage_by_request": ({"model": "model", "usage": {"prompt_tokens": 2}},) * 3}, + } + callback: Final = asyncio.create_task(captured_callback_kwargs(kwargs, response, NOW, NOW)) + try: + await entered.wait() + assert callback.done(), "Batch persistence must not consume the legacy callback deadline" + captured: Final = (await callback)["_accounting_capture"] + assert isinstance(captured, CapturedUsage) and not captured.receipt.done() + second: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW) + assert isinstance(second.get("_accounting_capture"), CapturedUsage) + overflow: Final = await captured_callback_kwargs(kwargs, response, NOW, NOW) + assert "_accounting_capture" not in overflow + assert "Batch capture producer capacity exhausted" in caplog.text + if outcome != "cancel": + release.set() + await runtime.stop(0 if outcome == "cancel" else 1) + assert captured.receipt.done() + assert captured.receipt.cancelled() is (outcome == "cancel") + if outcome == "failure": + assert isinstance(captured.receipt.exception(), RuntimeError) + assert "Batch usage capture failed" in caplog.text + assert len(database.rows) == (4 if outcome == "drain" else 0) + with pytest.raises(UsageCaptureError, match="active ledger writer"): + runtime.submit_batch(kwargs, response, NOW, NOW) + finally: + release.set() + await asyncio.gather(callback, return_exceptions=True) + await runtime.stop(1) diff --git a/tests/unit/proxy/spend_tracking/test_usage_meter.py b/tests/unit/proxy/spend_tracking/test_usage_meter.py index 2bd50bf7fca..f395c3b687d 100644 --- a/tests/unit/proxy/spend_tracking/test_usage_meter.py +++ b/tests/unit/proxy/spend_tracking/test_usage_meter.py @@ -427,3 +427,38 @@ async def test_cancelling_bounded_capture_preserves_admitted_writes_without_admi await meter.close() assert database.committed == ("0", "1") assert meter.pending_bytes == meter.pending_records == 0 + + +@pytest.mark.asyncio +async def test_capacity_wait_retries_admission_before_other_producers_finish() -> None: + entered: Final = asyncio.Queue[tuple[str, ...]]() + release: Final = asyncio.Queue[None]() + + async def persist(writes: tuple[UsageWrite, ...]) -> None: + entered.put_nowait(tuple(write.event.event_id for write in writes)) + await release.get() + + meter: Final = DurableUsageMeter( + persist, UsageCaptureSettings(batch_size=2, max_pending_records=3), wait_for_batch=_manual_flush + ) + initial: Final = meter.submit((_write("first"), _write("second"))) + remaining: Final = meter.submit((_write("remaining"),)) + assert await entered.get() == ("first", "second") + recording: Final = asyncio.create_task(meter.record_writes((_write("target"),))) + try: + await asyncio.sleep(0) + release.put_nowait(None) + assert await entered.get() == ("remaining",) + await asyncio.sleep(0) + assert meter.pending_records == 2 + release.put_nowait(None) + assert await entered.get() == ("target",) + release.put_nowait(None) + await recording + await asyncio.gather(initial, remaining) + finally: + recording.cancel() + await asyncio.gather(recording, return_exceptions=True) + release.put_nowait(None) + release.put_nowait(None) + await meter.close()