From 5a7a889d933a465cb12c94e349d7648bb3718891 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 13 Aug 2025 19:12:39 -0700 Subject: [PATCH] perf(main.py): new 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag improves RPS for openai calls by 100 (100 users, 10 start-up) Moves to using litellm's asynchttphandler vs. openais's sdk for llm calling --- .gitignore | 1 + litellm/litellm_core_utils/litellm_logging.py | 33 +++++----- litellm/main.py | 63 +++++++++++++------ litellm/proxy/_new_secret_config.yaml | 1 - 4 files changed, 60 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index f8d028ff47b..a58dae81ead 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql litellm/proxy/db/migrations/* litellm/proxy/migrations/*config.yaml litellm/proxy/migrations/* +litellm/proxy/to_delete_loadtest_work/* config.yaml tests/litellm/litellm_core_utils/llm_cost_calc/log.txt tests/test_custom_dir/* diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 8e4aa43c9ce..385df83c904 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -811,7 +811,7 @@ class Logging(LiteLLMLoggingBaseClass): str(e) ) ) - if self.logger_fn and callable(self.logger_fn): + if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: self.logger_fn( self.model_call_details @@ -999,7 +999,7 @@ class Logging(LiteLLMLoggingBaseClass): ) ) ) - if self.logger_fn and callable(self.logger_fn): + if getattr(self, "logger_fn", None) and callable(self.logger_fn): try: self.logger_fn( self.model_call_details @@ -3919,10 +3919,12 @@ class StandardLoggingPayloadSetup: # Generate cold storage object key if cold storage is configured if start_time is not None and response_id is not None: - cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key( - start_time=start_time, - response_id=response_id, - team_alias=clean_metadata.get("user_api_key_team_alias"), + cold_storage_object_key = ( + StandardLoggingPayloadSetup._generate_cold_storage_object_key( + start_time=start_time, + response_id=response_id, + team_alias=clean_metadata.get("user_api_key_team_alias"), + ) ) if cold_storage_object_key: clean_metadata["cold_storage_object_key"] = cold_storage_object_key @@ -4093,12 +4095,12 @@ class StandardLoggingPayloadSetup: ) -> Optional[str]: """ Generate cold storage object key in the same format as S3Logger. - + Args: start_time: The start time of the request - response_id: The response ID + response_id: The response ID team_alias: Optional team alias for team-based prefixing - + Returns: Optional[str]: The generated object key or None if cold storage not configured """ @@ -4112,26 +4114,23 @@ class StandardLoggingPayloadSetup: ColdStorageHandler._get_configured_cold_storage_custom_logger() ) except Exception as e: - verbose_logger.debug( - f"Cold storage custom logger unavailable: {e}" - ) + verbose_logger.debug(f"Cold storage custom logger unavailable: {e}") return None if configured_cold_storage_logger is None: return None - + try: # Generate file name in same format as litellm.utils.get_logging_id s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}" - s3_object_key = get_s3_object_key( - s3_path="", # Use empty path as default - team_alias_prefix="", # Don't split by team alias for cold storage + s3_path="", # Use empty path as default + team_alias_prefix="", # Don't split by team alias for cold storage start_time=start_time, s3_file_name=s3_file_name, ) - + return s3_object_key except Exception: # If any error occurs in generating the key, return None diff --git a/litellm/main.py b/litellm/main.py index 339d9e14406..47d2c82888c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -77,7 +77,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.realtime_api.main import _realtime_health_check -from litellm.secret_managers.main import get_secret_str +from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import RawRequestTypedDict from litellm.utils import ( @@ -1981,26 +1981,49 @@ def completion( # type: ignore # noqa: PLR0915 optional_params[k] = v ## COMPLETION CALL + use_base_llm_http_handler = get_secret_bool( + "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER" + ) try: - response = openai_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, - custom_llm_provider=custom_llm_provider, - ) + if use_base_llm_http_handler: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=encoding, + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + else: + response = openai_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, + custom_llm_provider=custom_llm_provider, + ) except Exception as e: ## LOGGING - log the original exception returned logging.post_call( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 99460a0547a..b48fe5be1c3 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -6,7 +6,6 @@ model_list: api_base: https://exampleopenaiendpoint-production.up.railway.app/ litellm_settings: - callbacks: ["otel"] cache: true cache_params: type: redis