From d01efcb3084a0a3bd7302294d1072ab8ab247ecd Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 15 Nov 2025 15:58:41 -0800 Subject: [PATCH 01/12] speech set up --- no_cache_hits.py | 48 +++++++++++++++++++++++++++++++++++++++++++++ speech.mp3 | Bin 0 -> 104 bytes speech_config.yaml | 9 +++++++++ 3 files changed, 57 insertions(+) create mode 100644 no_cache_hits.py create mode 100644 speech.mp3 create mode 100644 speech_config.yaml diff --git a/no_cache_hits.py b/no_cache_hits.py new file mode 100644 index 00000000000..1b3bf895f77 --- /dev/null +++ b/no_cache_hits.py @@ -0,0 +1,48 @@ +from locust import HttpUser, between, task + + +class MyUser(HttpUser): + """ + Minimal Locust user for repeatedly hitting `/v1/audio/speech`. + The goal is to measure server-side performance, so we avoid any extra work + (file writes, random generation, manual timing, custom event hooks, etc.) + that could inflate client-side latency. + """ + + wait_time = between(0.5, 1) + host = "http://0.0.0.0:8090" + + def on_start(self): + self.api_key = "sk-1234" + self.model_name = "fake-openai-speech" + self.headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + self.prompt_counter = 0 + + @task + def audio_speech_request(self): + self.prompt_counter += 1 + # Ensure prompts differ slightly so the backend can't reuse cached audio. + prompt = ( + "Generate a short spoken status update mentioning counter " + f"{self.prompt_counter}." + ) + + response = self.client.post( + "v1/audio/speech", + json={ + "model": self.model_name, + "input": prompt, + "voice": "alloy", + "format": "mp3", + }, + headers=self.headers, + name="audio_speech", + ) + + if response.status_code != 200: + # log the errors in error.txt + with open("error.txt", "a") as error_log: + error_log.write(response.text + "\n") \ No newline at end of file diff --git a/speech.mp3 b/speech.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..f4f854d9bd215e2493d48b4bc4d39804bd79c038 GIT binary patch literal 104 NcmezWdjbPJ000JT0*e3u literal 0 HcmV?d00001 diff --git a/speech_config.yaml b/speech_config.yaml new file mode 100644 index 00000000000..ad9920a2793 --- /dev/null +++ b/speech_config.yaml @@ -0,0 +1,9 @@ +model_list: + - model_name: fake-openai-speech + litellm_params: + model: openai/gpt-4o-mini-tts + api_base: http://0.0.0.0:8090/ + api_key: sk-1234 + model_info: + mode: audio_speech + \ No newline at end of file From 44f2013495c6987bef3918ca045f942777b21c34 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 15 Nov 2025 17:02:15 -0800 Subject: [PATCH 02/12] fix: change chunk_size for aiter_bytes 1KB is too small for audio and is lowering the RPS when testing with medium to large files --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a6e73199f0e..36178652a7d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5305,7 +5305,7 @@ async def audio_speech( # Printing each chunk size async def generate(_response: HttpxBinaryResponseContent): - _generator = await _response.aiter_bytes(chunk_size=1024) + _generator = await _response.aiter_bytes(chunk_size=4096) async for chunk in _generator: yield chunk From 348d28d871a8c8d00ec1263d6d61caff4843cd7b Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 15 Nov 2025 17:15:26 -0800 Subject: [PATCH 03/12] fix: remove function definition from every request --- litellm/proxy/proxy_server.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 36178652a7d..c65c7f77f0d 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14,6 +14,7 @@ from datetime import datetime, timedelta from typing import ( TYPE_CHECKING, Any, + AsyncGenerator, List, Literal, Optional, @@ -5231,6 +5232,14 @@ async def moderations( ) +async def _audio_speech_chunk_generator( + _response: HttpxBinaryResponseContent, +) -> AsyncGenerator[bytes, None]: + _generator = await _response.aiter_bytes(chunk_size=4096) + async for chunk in _generator: + yield chunk + + @router.post( "/v1/audio/speech", dependencies=[Depends(user_api_key_auth)], @@ -5303,12 +5312,6 @@ async def audio_speech( response_cost = hidden_params.get("response_cost", None) or "" litellm_call_id = hidden_params.get("litellm_call_id", None) or "" - # Printing each chunk size - async def generate(_response: HttpxBinaryResponseContent): - _generator = await _response.aiter_bytes(chunk_size=4096) - async for chunk in _generator: - yield chunk - custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers( user_api_key_dict=user_api_key_dict, model_id=model_id, @@ -5337,7 +5340,9 @@ async def audio_speech( media_type = "audio/wav" # Gemini TTS returns WAV format after conversion return StreamingResponse( - generate(response), media_type=media_type, headers=custom_headers # type: ignore + _audio_speech_chunk_generator(response), # type: ignore[arg-type] + media_type=media_type, + headers=custom_headers, # type: ignore ) except Exception as e: From 8ea0e31678863d2d700bf857bedac4d25338e008 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 15 Nov 2025 17:30:58 -0800 Subject: [PATCH 04/12] Optimize streaming response accumulation Refactor async_data_generator to build streamed text via list accumulation and ''.join() instead of repeated string concatenation. This improves performance for long responses without changing streaming behavior. --- litellm/proxy/proxy_server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index c65c7f77f0d..b67de4a87a7 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4017,7 +4017,8 @@ async def async_data_generator( ): verbose_proxy_logger.debug("inside generator") try: - str_so_far = "" + # Use a list to accumulate response segments to avoid O(n^2) string concatenation + str_so_far_parts: list[str] = [] error_message: Optional[str] = None async for chunk in proxy_logging_obj.async_post_call_streaming_iterator_hook( user_api_key_dict=user_api_key_dict, @@ -4033,12 +4034,12 @@ async def async_data_generator( user_api_key_dict=user_api_key_dict, response=chunk, data=request_data, - str_so_far=str_so_far, + str_so_far="".join(str_so_far_parts), ) if isinstance(chunk, (ModelResponse, ModelResponseStream)): response_str = litellm.get_response_string(response_obj=chunk) - str_so_far += response_str + str_so_far_parts.append(response_str) if isinstance(chunk, BaseModel): chunk = chunk.model_dump_json(exclude_none=True, exclude_unset=True) From 98e2b64040f6e5f882648b433a23799582d710a1 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Sat, 15 Nov 2025 17:37:01 -0800 Subject: [PATCH 05/12] Optimize response string construction Use list accumulation and join in get_response_string to avoid O(n^2) string concatenation and add a brief comment explaining the performance rationale. --- litellm/utils.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 783d462a7af..201e8145254 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4437,17 +4437,20 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream]) responses_api_response = getattr(response_obj, "response", None) if responses_api_response and hasattr(responses_api_response, "output"): output_list = responses_api_response.output - response_str = "" + # Use list accumulation to avoid O(n^2) string concatenation: + # repeatedly doing `response_str += part` copies the full string each time + # because Python strings are immutable, so total work grows with n^2. + response_output_parts: List[str] = [] for output_item in output_list: # Handle output items with content array if hasattr(output_item, "content"): for content_part in output_item.content: if hasattr(content_part, "text"): - response_str += content_part.text + response_output_parts.append(content_part.text) # Handle output items with direct text field elif hasattr(output_item, "text"): - response_str += output_item.text - return response_str + response_output_parts.append(output_item.text) + return "".join(response_output_parts) # Handle Responses API text delta events if hasattr(response_obj, "type") and hasattr(response_obj, "delta"): @@ -4461,16 +4464,17 @@ def get_response_string(response_obj: Union[ModelResponse, ModelResponseStream]) response_obj.choices ) - response_str = "" + # Use list accumulation to avoid O(n^2) string concatenation across choices + response_parts: List[str] = [] for choice in _choices: if isinstance(choice, Choices): if choice.message.content is not None: - response_str += choice.message.content + response_parts.append(str(choice.message.content)) elif isinstance(choice, StreamingChoices): if choice.delta.content is not None: - response_str += choice.delta.content + response_parts.append(str(choice.delta.content)) - return response_str + return "".join(response_parts) def get_api_key(llm_provider: str, dynamic_api_key: Optional[str]): From 4614e528dc4b3385582810f3c565d62668373d3c Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Mon, 17 Nov 2025 10:11:55 -0800 Subject: [PATCH 06/12] fix: remove deadcode The optimizations related to `select_data_generator` had no effect because its output which is the generator wasn't being used. --- litellm/proxy/proxy_server.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b67de4a87a7..d6a98a1ca5b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5327,11 +5327,6 @@ async def audio_speech( hidden_params=hidden_params, ) - select_data_generator( - response=response, - user_api_key_dict=user_api_key_dict, - request_data=data, - ) # Determine media type based on model type media_type = "audio/mpeg" # Default for OpenAI TTS request_model = data.get("model", "") From c8c12298590885bc845088d4982c7059996f04a9 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Mon, 17 Nov 2025 10:24:34 -0800 Subject: [PATCH 07/12] fix: call_type mistake & remove repetitive .lower() calls --- litellm/proxy/proxy_server.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d6a98a1ca5b..cb43ef0cecf 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5286,7 +5286,7 @@ async def audio_speech( ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook( - user_api_key_dict=user_api_key_dict, data=data, call_type="image_generation" + user_api_key_dict=user_api_key_dict, data=data, call_type="aspeech" ) ## ROUTE TO CORRECT ENDPOINT ## @@ -5330,10 +5330,12 @@ async def audio_speech( # Determine media type based on model type media_type = "audio/mpeg" # Default for OpenAI TTS request_model = data.get("model", "") - if "gemini" in request_model.lower() and ( - "tts" in request_model.lower() or "preview-tts" in request_model.lower() - ): - media_type = "audio/wav" # Gemini TTS returns WAV format after conversion + if request_model: + request_model_lower = request_model.lower() + if "gemini" in request_model_lower and ( + "tts" in request_model_lower or "preview-tts" in request_model_lower + ): + media_type = "audio/wav" # Gemini TTS returns WAV format after conversion return StreamingResponse( _audio_speech_chunk_generator(response), # type: ignore[arg-type] From 697cb0906011cb84f2cc8c34b4ff7a2d7402dc13 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Mon, 17 Nov 2025 10:51:18 -0800 Subject: [PATCH 08/12] fix: shared_sessions not being used --- litellm/llms/openai/openai.py | 5 +++++ litellm/main.py | 2 ++ 2 files changed, 7 insertions(+) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 2949e35e5e7..3282b7665c0 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1414,6 +1414,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): timeout: Union[float, httpx.Timeout], aspeech: Optional[bool] = None, client=None, + shared_session: Optional["ClientSession"] = None, ) -> HttpxBinaryResponseContent: if aspeech is not None and aspeech is True: return self.async_audio_speech( @@ -1428,6 +1429,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries=max_retries, timeout=timeout, client=client, + shared_session=shared_session, ) # type: ignore openai_client = self._get_openai_client( @@ -1437,6 +1439,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): timeout=timeout, max_retries=max_retries, client=client, + shared_session=shared_session, ) response = cast(OpenAI, openai_client).audio.speech.create( @@ -1460,6 +1463,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): max_retries: int, timeout: Union[float, httpx.Timeout], client=None, + shared_session: Optional["ClientSession"] = None, ) -> HttpxBinaryResponseContent: openai_client = cast( AsyncOpenAI, @@ -1470,6 +1474,7 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): timeout=timeout, max_retries=max_retries, client=client, + shared_session=shared_session, ), ) diff --git a/litellm/main.py b/litellm/main.py index 14d0b04b7b5..412d7f1c38e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5747,6 +5747,7 @@ def speech( # noqa: PLR0915 proxy_server_request = kwargs.get("proxy_server_request", None) extra_headers = kwargs.get("extra_headers", None) model_info = kwargs.get("model_info", None) + shared_session = kwargs.get("shared_session", None) model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, api_base=api_base ) # type: ignore @@ -5856,6 +5857,7 @@ def speech( # noqa: PLR0915 timeout=timeout, client=client, # pass AsyncOpenAI, OpenAI client aspeech=aspeech, + shared_session=shared_session, ) elif custom_llm_provider == "azure": # Check if this is Azure Speech Service (Cognitive Services TTS) From f1895265e6b5643ef0e5be97b6f4cd65fcc2e78f Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Mon, 17 Nov 2025 12:55:58 -0800 Subject: [PATCH 09/12] fix: increase chunk_size to 8 KB for optimal latency --- litellm/proxy/proxy_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cb43ef0cecf..16988205530 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5236,7 +5236,7 @@ async def moderations( async def _audio_speech_chunk_generator( _response: HttpxBinaryResponseContent, ) -> AsyncGenerator[bytes, None]: - _generator = await _response.aiter_bytes(chunk_size=4096) + _generator = await _response.aiter_bytes(chunk_size=8192) async for chunk in _generator: yield chunk From 7241b4e9b505ea433563a88916cece2d062cd063 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Mon, 17 Nov 2025 12:59:48 -0800 Subject: [PATCH 10/12] add: comment above optimization For anybody that would change this value for whatever reason, the comment makes the tradeoff clear. --- litellm/proxy/proxy_server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 16988205530..da592c90720 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5236,6 +5236,10 @@ async def moderations( async def _audio_speech_chunk_generator( _response: HttpxBinaryResponseContent, ) -> AsyncGenerator[bytes, None]: + # chunk_size has a big impact on latency, it can't be too small or too large + # too small: latency is high + # too large: latency is low, but memory usage is high + # 8192 is a good compromise _generator = await _response.aiter_bytes(chunk_size=8192) async for chunk in _generator: yield chunk From b4e25a68a4690e93110fb4ca2c4aa56f1c08a25a Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Tue, 18 Nov 2025 09:40:26 -0800 Subject: [PATCH 11/12] fix: remove test files --- no_cache_hits.py | 48 --------------------------------------------- speech.mp3 | Bin 104 -> 0 bytes speech_config.yaml | 9 --------- 3 files changed, 57 deletions(-) delete mode 100644 no_cache_hits.py delete mode 100644 speech.mp3 delete mode 100644 speech_config.yaml diff --git a/no_cache_hits.py b/no_cache_hits.py deleted file mode 100644 index 1b3bf895f77..00000000000 --- a/no_cache_hits.py +++ /dev/null @@ -1,48 +0,0 @@ -from locust import HttpUser, between, task - - -class MyUser(HttpUser): - """ - Minimal Locust user for repeatedly hitting `/v1/audio/speech`. - The goal is to measure server-side performance, so we avoid any extra work - (file writes, random generation, manual timing, custom event hooks, etc.) - that could inflate client-side latency. - """ - - wait_time = between(0.5, 1) - host = "http://0.0.0.0:8090" - - def on_start(self): - self.api_key = "sk-1234" - self.model_name = "fake-openai-speech" - self.headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json", - } - self.prompt_counter = 0 - - @task - def audio_speech_request(self): - self.prompt_counter += 1 - # Ensure prompts differ slightly so the backend can't reuse cached audio. - prompt = ( - "Generate a short spoken status update mentioning counter " - f"{self.prompt_counter}." - ) - - response = self.client.post( - "v1/audio/speech", - json={ - "model": self.model_name, - "input": prompt, - "voice": "alloy", - "format": "mp3", - }, - headers=self.headers, - name="audio_speech", - ) - - if response.status_code != 200: - # log the errors in error.txt - with open("error.txt", "a") as error_log: - error_log.write(response.text + "\n") \ No newline at end of file diff --git a/speech.mp3 b/speech.mp3 deleted file mode 100644 index f4f854d9bd215e2493d48b4bc4d39804bd79c038..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 104 NcmezWdjbPJ000JT0*e3u diff --git a/speech_config.yaml b/speech_config.yaml deleted file mode 100644 index ad9920a2793..00000000000 --- a/speech_config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -model_list: - - model_name: fake-openai-speech - litellm_params: - model: openai/gpt-4o-mini-tts - api_base: http://0.0.0.0:8090/ - api_key: sk-1234 - model_info: - mode: audio_speech - \ No newline at end of file From 1c67b7e1daf28a9d9afc24d647adfb35c71d322b Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Thu, 20 Nov 2025 17:48:40 -0800 Subject: [PATCH 12/12] fix: place hardcoded value on constants.py --- litellm/constants.py | 1 + litellm/proxy/proxy_server.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/litellm/constants.py b/litellm/constants.py index 3f763cad926..fc26e1cf817 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -246,6 +246,7 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) +AUDIO_SPEECH_CHUNK_SIZE = 8192 # chunk_size for audio speech streaming. Balance between latency and memory usage MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index da592c90720..47c30e9ce84 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -32,6 +32,7 @@ from litellm.constants import ( AIOHTTP_CONNECTOR_LIMIT, AIOHTTP_KEEPALIVE_TIMEOUT, AIOHTTP_TTL_DNS_CACHE, + AUDIO_SPEECH_CHUNK_SIZE, BASE_MCP_ROUTE, DEFAULT_MAX_RECURSE_DEPTH, DEFAULT_SHARED_HEALTH_CHECK_LOCK_TTL, @@ -5240,7 +5241,7 @@ async def _audio_speech_chunk_generator( # too small: latency is high # too large: latency is low, but memory usage is high # 8192 is a good compromise - _generator = await _response.aiter_bytes(chunk_size=8192) + _generator = await _response.aiter_bytes(chunk_size=AUDIO_SPEECH_CHUNK_SIZE) async for chunk in _generator: yield chunk