scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

Merge pull request #14816 from leventov/no-streaming-logging

Browse source 
Fix concurrency/scaling when many Python threads do streaming using *sync* completions
This commit is contained in:
Krish Dholakia 2025-09-23 22:53:18 -07:00 • committed by GitHub
commit 2e3e7de2eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/litellm_core_utils/streaming_handler.py
View file

@ -1619,11 +1619,12 @@ class CustomStreamWrapper:
completion_start_time=datetime.datetime.now()
)
## LOGGING
executor.submit(
self.run_success_logging_and_cache_storage,
response,
cache_hit,
) # log response
if not litellm.disable_streaming_logging:
executor.submit(
self.run_success_logging_and_cache_storage,
response,
cache_hit,
) # log response
choice = response.choices[0]
if isinstance(choice, StreamingChoices):
self.response_uptil_now += choice.delta.get("content", "") or ""

2
litellm/llms/custom_httpx/http_handler.py
View file

@ -935,7 +935,7 @@ class HTTPHandler:
if litellm.force_ipv4:
return HTTPTransport(local_address="0.0.0.0")
else:
return None
return getattr(litellm, 'sync_transport', None)
def get_async_httpx_client(