scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

fix: bound spend_log_transactions queue to prevent unbounded memory growth

Browse source 
When DB writes are slow or failing, the spend_log_transactions list grows
without bound. Each entry is a full SpendLogsPayload dict. Under 2000 RPS
with 10s DB lag, this is 20,000 entries accumulating indefinitely.

Add MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE (default 10000, configurable via
env var). When the queue exceeds this limit, the oldest 10% of entries are
dropped and a warning is logged.

Also simplify the duplicate if/elif branches in _insert_spend_log_to_db
(both with and without spend_logs_url did the same append).

Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-03-13 22:58:15 +00:00
parent ecd2525748
commit c873abcda2
No known key found for this signature in database
2 changed files with 22 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
litellm/constants.py
View file

@ -1344,6 +1344,11 @@ SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
# Maximum number of spend log entries to hold in-memory before dropping oldest.
# Prevents unbounded memory growth when DB writes are slow or failing.
MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE = int(
os.getenv("MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE", 10000)
)
DEFAULT_CRON_JOB_LOCK_TTL_SECONDS = int(
os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60)
) # 1 minute

21
litellm/proxy/db/db_spend_update_writer.py
View file

@ -720,16 +720,29 @@ class DBSpendUpdateWriter:
prisma_client: Optional[PrismaClient] = None,
spend_logs_url: Optional[str] = os.getenv("SPEND_LOGS_URL"),
) -> Optional[PrismaClient]:
from litellm.constants import MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE
verbose_proxy_logger.debug(
"Writing spend log to db - request_id: {}, spend: {}".format(
payload.get("request_id"), payload.get("spend")
)
)
if prisma_client is not None and spend_logs_url is not None:
async with prisma_client._spend_log_transactions_lock:
prisma_client.spend_log_transactions.append(payload)
elif prisma_client is not None:
if prisma_client is not None:
async with prisma_client._spend_log_transactions_lock:
# Prevent unbounded queue growth when DB writes are slow/failing
if (
len(prisma_client.spend_log_transactions)
>= MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE
):
drop_count = MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE // 10
prisma_client.spend_log_transactions = (
prisma_client.spend_log_transactions[drop_count:]
)
verbose_proxy_logger.warning(
"Spend log queue exceeded %d entries, dropped oldest %d to prevent memory leak",
MAX_SPEND_LOG_TRANSACTIONS_QUEUE_SIZE,
drop_count,
)
prisma_client.spend_log_transactions.append(payload)
else:
verbose_proxy_logger.debug(