mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(accounting): isolate invalid producer submissions within usage batches
This commit is contained in:
parent
ff3c6b632f
commit
4ff4538bac
3 changed files with 81 additions and 5 deletions
|
|
@ -87,6 +87,15 @@ def _retryable_persistence_failure(error: Exception) -> bool:
|
|||
)
|
||||
|
||||
|
||||
def _isolatable_persistence_failure(error: Exception) -> bool:
|
||||
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
|
||||
|
||||
return isinstance(error, AccountingConflictError) or any(
|
||||
isinstance(cause, Exception) and (PrismaDBExceptionHandler.postgres_sqlstate(cause) or "").startswith("22")
|
||||
for cause in exception_chain(error)
|
||||
)
|
||||
|
||||
|
||||
class DurableUsageMeter:
|
||||
def __init__(
|
||||
self,
|
||||
|
|
@ -192,8 +201,8 @@ class DurableUsageMeter:
|
|||
except asyncio.CancelledError:
|
||||
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
|
||||
raise
|
||||
except AccountingConflictError:
|
||||
if len(batch) > 1:
|
||||
except Exception as error: # noqa: BLE001 # isolate data faults, surface systemic failures to all producers
|
||||
if len(batch) > 1 and _isolatable_persistence_failure(error):
|
||||
try:
|
||||
for item in batch:
|
||||
await self._commit((item,))
|
||||
|
|
@ -201,9 +210,12 @@ class DurableUsageMeter:
|
|||
self._finish(batch, UsageCaptureError("Usage commit acknowledgement interrupted"))
|
||||
raise
|
||||
return
|
||||
self._finish(batch, UsageCaptureError("Usage identity conflicts with accepted content"))
|
||||
except Exception as error: # noqa: BLE001 # persistence failures must reach every producer receipt
|
||||
self._finish(batch, error)
|
||||
self._finish(
|
||||
batch,
|
||||
UsageCaptureError("Usage identity conflicts with accepted content")
|
||||
if isinstance(error, AccountingConflictError)
|
||||
else error,
|
||||
)
|
||||
else:
|
||||
self._finish(batch, None)
|
||||
|
||||
|
|
|
|||
|
|
@ -576,3 +576,30 @@ async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
|
|||
await store.insert_usage_batch((*writes, *writes))
|
||||
assert _rows(ledger_database, 'SELECT * FROM "LiteLLM_AccountingDelivery" ORDER BY event_id, target') == before
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingEvent"') == [(2,)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_jsonb_submission_cannot_reject_another_producers_commit(
|
||||
writer: _Writer, ledger_database: str
|
||||
) -> None:
|
||||
from prisma.errors import DataError
|
||||
|
||||
from litellm.proxy.spend_tracking.usage_meter import DurableUsageMeter, UsageCaptureSettings
|
||||
|
||||
_, store = writer
|
||||
invalid: Final = _usage("invalid")
|
||||
valid: Final = _usage("valid")
|
||||
meter: Final = DurableUsageMeter(store.insert_usage_batch, UsageCaptureSettings(batch_size=2))
|
||||
rejected: Final = meter.submit(
|
||||
(UsageWrite(invalid, _envelope(invalid, context={"parent": "bad\x00text"}), _TARGETS),)
|
||||
)
|
||||
accepted: Final = meter.submit((UsageWrite(valid, _envelope(valid), _TARGETS),))
|
||||
try:
|
||||
with pytest.raises(DataError):
|
||||
await rejected
|
||||
await accepted
|
||||
finally:
|
||||
await meter.close()
|
||||
assert _rows(ledger_database, 'SELECT event_id FROM "LiteLLM_AccountingEvent"') == [("valid",)]
|
||||
assert _rows(ledger_database, 'SELECT count(*) FROM "LiteLLM_AccountingDelivery"') == [(len(_TARGETS),)]
|
||||
assert (meter.pending_bytes, meter.pending_records) == (0, 0)
|
||||
|
|
|
|||
|
|
@ -302,3 +302,40 @@ async def test_burst_drains_full_batches_without_waiting_for_partial_batch_deadl
|
|||
finally:
|
||||
await meter.stop(0)
|
||||
await asyncio.gather(*receipts, return_exceptions=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("sqlstate", ["22P05", "22003", "42P01"])
|
||||
async def test_invalid_submission_isolated_without_retrying_systemic_faults(sqlstate: str) -> None:
|
||||
from prisma.errors import DataError
|
||||
|
||||
database: Final = CommitBarrier()
|
||||
database.release.set()
|
||||
attempts: Final = asyncio.Queue[tuple[str, ...]]()
|
||||
|
||||
async def persist(writes: tuple[UsageWrite, ...]) -> None:
|
||||
identities: Final = tuple(write.event.event_id for write in writes)
|
||||
attempts.put_nowait(identities)
|
||||
if "invalid" in identities or sqlstate == "42P01":
|
||||
raise DataError(
|
||||
data={"user_facing_error": {"message": "database rejected input", "meta": {"code": sqlstate}}}
|
||||
)
|
||||
await database.write(writes)
|
||||
|
||||
meter: Final = DurableUsageMeter(persist, UsageCaptureSettings(batch_size=3))
|
||||
rejected: Final = meter.submit((_write("caller"), _write("invalid")))
|
||||
neighbor: Final = meter.submit((_write("neighbor"),))
|
||||
try:
|
||||
with pytest.raises(DataError):
|
||||
await rejected
|
||||
if sqlstate == "42P01":
|
||||
with pytest.raises(DataError):
|
||||
await neighbor
|
||||
assert attempts.qsize() == 1
|
||||
else:
|
||||
await neighbor
|
||||
assert database.committed == ("neighbor",)
|
||||
assert attempts.qsize() == 3
|
||||
finally:
|
||||
await meter.close()
|
||||
assert (meter.pending_records, meter.pending_bytes) == (0, 0)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue