scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix(streaming): handle RuntimeError during model_copy in streaming handler

Browse source 
The race condition occurs when model_copy(deep=True) tries to deepcopy
_hidden_params dict while it's being concurrently modified by logging
callbacks. Fall back to shallow copy if the deep copy fails.

Co-authored-by: Ishaan Jaff <ishaan-jaff@users.noreply.github.com>
This commit is contained in:
Cursor Agent 2026-03-19 02:43:06 +00:00
parent d56f22a85c
commit 1e22385479
No known key found for this signature in database
1 changed files with 19 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1893,15 +1893,23 @@ class CustomStreamWrapper:
"usage",
getattr(complete_streaming_response, "usage"),
)
self.cache_streaming_response(
processed_chunk=complete_streaming_response.model_copy(
try:
_cache_copy = complete_streaming_response.model_copy(
deep=True
),
)
_log_copy = complete_streaming_response.model_copy(
deep=True
)
except RuntimeError:
_cache_copy = complete_streaming_response.model_copy()
_log_copy = complete_streaming_response.model_copy()
self.cache_streaming_response(
processed_chunk=_cache_copy,
cache_hit=cache_hit,
)
executor.submit(
self.logging_obj.success_handler,
complete_streaming_response.model_copy(deep=True),
_log_copy,
None,
None,
cache_hit,
@ -2113,11 +2121,15 @@ class CustomStreamWrapper:
"usage",
getattr(complete_streaming_response, "usage"),
)
try:
_copy = complete_streaming_response.model_copy(
deep=True
)
except RuntimeError:
_copy = complete_streaming_response.model_copy()
asyncio.create_task(
self.async_cache_streaming_response(
processed_chunk=complete_streaming_response.model_copy(
deep=True
),
processed_chunk=_copy,
cache_hit=cache_hit,
)
)