scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-10 22:41:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 82

fix: place hardcoded value on constants.py

Browse source
This commit is contained in:
AlexsanderHamir 2025-11-20 17:48:40 -08:00
parent b4e25a68a4
commit 1c67b7e1da
2 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm/constants.py
View file

@ -246,6 +246,7 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350))
QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99))
QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536))
CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02))
AUDIO_SPEECH_CHUNK_SIZE = 8192 # chunk_size for audio speech streaming. Balance between latency and memory usage
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
)

3
litellm/proxy/proxy_server.py
View file

@ -32,6 +32,7 @@ from litellm.constants import (
AIOHTTP_CONNECTOR_LIMIT,
AIOHTTP_KEEPALIVE_TIMEOUT,
AIOHTTP_TTL_DNS_CACHE,
AUDIO_SPEECH_CHUNK_SIZE,
BASE_MCP_ROUTE,
DEFAULT_MAX_RECURSE_DEPTH,
DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL,
@ -5240,7 +5241,7 @@ async def _audio_speech_chunk_generator(
# too small: latency is high
# too large: latency is low, but memory usage is high
# 8192 is a good compromise
_generator = await _response.aiter_bytes(chunk_size=8192)
_generator = await _response.aiter_bytes(chunk_size=AUDIO_SPEECH_CHUNK_SIZE)
async for chunk in _generator:
yield chunk