scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 83

fix: reduce deepcopy in spend tracking from 3x to 1x per request

Browse source 
The spend tracking path previously called copy.deepcopy(payload) three
times per request:
1. payload_copy for daily spend helpers
2. request_tags deepcopy
3. payload for _insert_spend_log_to_db

Analysis shows daily spend helpers only READ from the payload (extracting
numeric fields like spend, tokens, user_id). They never mutate it.
request_tags is a list of strings that only needs a shallow copy.

Now: only 1 deepcopy for the spend log queue (which is consumed async
by a background job), and 0 copies when disable_spend_logs=True.

For 380KB payloads at 2000 RPS, this eliminates ~1.5 GB/s of allocation
churn that was being generated solely for GC to clean up.

Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-03-13 22:59:28 +00:00
parent c873abcda2
commit 2bf755f1ad
No known key found for this signature in database
1 changed files with 15 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
litellm/proxy/db/db_spend_update_writer.py
View file

@ -146,16 +146,16 @@ class DBSpendUpdateWriter:
if team_id is not None and team_id != "":
payload["team_id"] = team_id
# One deepcopy shared by all 6 daily spend helpers (was 5, fixes agent bug)
payload_copy = copy.deepcopy(payload)
# Deepcopy request_tags for _update_tag_db
request_tags = copy.deepcopy(payload.get("request_tags"))
# Keep _insert_spend_log_to_db awaited inline (not a task, preserve current behavior)
# --- Memory optimization: reduce deepcopy from 3x to 1x per request ---
# The spend log queue (_insert_spend_log_to_db) needs its own copy
# because it's consumed asynchronously by a background job.
# The daily spend helpers in _batch_database_updates only READ
# from the payload (extracting numeric fields), so they can share
# the original dict without a copy.
if disable_spend_logs is False:
spend_log_payload = copy.deepcopy(payload)
await self._insert_spend_log_to_db(
payload=copy.deepcopy(payload),
payload=spend_log_payload,
prisma_client=prisma_client,
)
else:
@ -163,6 +163,12 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
# Daily spend helpers only read from the payload (no mutation),
# so no deepcopy is needed. Shallow-copy request_tags (list of strings).
request_tags = payload.get("request_tags")
if isinstance(request_tags, list):
request_tags = list(request_tags)
# Single task replaces 11 create_task() calls
asyncio.create_task(
self._batch_database_updates(
@ -175,7 +181,7 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
user_api_key_cache=user_api_key_cache,
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload_copy=payload_copy,
payload_copy=payload,
request_tags=request_tags,
)
)