Merge pull request #13625 from BerriAI/litellm_dev_08_13_2025_p1

perf(main.py): new 'EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER' flag

.gitignore

@@ -86,6 +86,7 @@ litellm/proxy/db/migrations/0_init/migration.sql
 litellm/proxy/db/migrations/*
 litellm/proxy/migrations/*config.yaml
 litellm/proxy/migrations/*
+litellm/proxy/to_delete_loadtest_work/*
 config.yaml
 tests/litellm/litellm_core_utils/llm_cost_calc/log.txt
 tests/test_custom_dir/*

litellm/litellm_core_utils/litellm_logging.py

@@ -811,7 +811,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             str(e)
                         )
                     )
-            if self.logger_fn and callable(self.logger_fn):
+            if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
                     self.logger_fn(
                         self.model_call_details
@@ -999,7 +999,7 @@ class Logging(LiteLLMLoggingBaseClass):
                         )
                     )
                 )
-            if self.logger_fn and callable(self.logger_fn):
+            if getattr(self, "logger_fn", None) and callable(self.logger_fn):
                 try:
                     self.logger_fn(
                         self.model_call_details
@@ -3919,10 +3919,12 @@ class StandardLoggingPayloadSetup:
 
         # Generate cold storage object key if cold storage is configured
         if start_time is not None and response_id is not None:
-            cold_storage_object_key = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
-                start_time=start_time,
-                response_id=response_id,
-                team_alias=clean_metadata.get("user_api_key_team_alias"),
+            cold_storage_object_key = (
+                StandardLoggingPayloadSetup._generate_cold_storage_object_key(
+                    start_time=start_time,
+                    response_id=response_id,
+                    team_alias=clean_metadata.get("user_api_key_team_alias"),
+                )
             )
             if cold_storage_object_key:
                 clean_metadata["cold_storage_object_key"] = cold_storage_object_key
@@ -4093,12 +4095,12 @@ class StandardLoggingPayloadSetup:
     ) -> Optional[str]:
         """
         Generate cold storage object key in the same format as S3Logger.
-        
+
         Args:
             start_time: The start time of the request
-            response_id: The response ID 
+            response_id: The response ID
             team_alias: Optional team alias for team-based prefixing
-            
+
         Returns:
             Optional[str]: The generated object key or None if cold storage not configured
         """
@@ -4112,26 +4114,23 @@ class StandardLoggingPayloadSetup:
                 ColdStorageHandler._get_configured_cold_storage_custom_logger()
             )
         except Exception as e:
-            verbose_logger.debug(
-                f"Cold storage custom logger unavailable: {e}"
-            )
+            verbose_logger.debug(f"Cold storage custom logger unavailable: {e}")
             return None
 
         if configured_cold_storage_logger is None:
             return None
-            
+
         try:
             # Generate file name in same format as litellm.utils.get_logging_id
             s3_file_name = f"time-{start_time.strftime('%H-%M-%S-%f')}_{response_id}"
 
-            
             s3_object_key = get_s3_object_key(
-                s3_path="",             # Use empty path as default
-                team_alias_prefix="",   # Don't split by team alias for cold storage
+                s3_path="",  # Use empty path as default
+                team_alias_prefix="",  # Don't split by team alias for cold storage
                 start_time=start_time,
                 s3_file_name=s3_file_name,
             )
-            
+
             return s3_object_key
         except Exception:
             # If any error occurs in generating the key, return None

litellm/main.py

@@ -77,7 +77,7 @@ from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig
 from litellm.llms.bedrock.common_utils import BedrockModelInfo
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.realtime_api.main import _realtime_health_check
-from litellm.secret_managers.main import get_secret_str
+from litellm.secret_managers.main import get_secret_bool, get_secret_str
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import RawRequestTypedDict
 from litellm.utils import (
@@ -1973,26 +1973,49 @@ def completion(  # type: ignore # noqa: PLR0915
                     optional_params[k] = v
 
             ## COMPLETION CALL
+            use_base_llm_http_handler = get_secret_bool(
+                "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER"
+            )
             try:
-                response = openai_chat_completions.completion(
-                    model=model,
-                    messages=messages,
-                    headers=headers,
-                    model_response=model_response,
-                    print_verbose=print_verbose,
-                    api_key=api_key,
-                    api_base=api_base,
-                    acompletion=acompletion,
-                    logging_obj=logging,
-                    optional_params=optional_params,
-                    litellm_params=litellm_params,
-                    logger_fn=logger_fn,
-                    timeout=timeout,  # type: ignore
-                    custom_prompt_dict=custom_prompt_dict,
-                    client=client,  # pass AsyncOpenAI, OpenAI client
-                    organization=organization,
-                    custom_llm_provider=custom_llm_provider,
-                )
+                if use_base_llm_http_handler:
+                    response = base_llm_http_handler.completion(
+                        model=model,
+                        messages=messages,
+                        api_base=api_base,
+                        custom_llm_provider=custom_llm_provider,
+                        model_response=model_response,
+                        encoding=encoding,
+                        logging_obj=logging,
+                        optional_params=optional_params,
+                        timeout=timeout,
+                        litellm_params=litellm_params,
+                        acompletion=acompletion,
+                        stream=stream,
+                        api_key=api_key,
+                        headers=headers,
+                        client=client,
+                        provider_config=provider_config,
+                    )
+                else:
+                    response = openai_chat_completions.completion(
+                        model=model,
+                        messages=messages,
+                        headers=headers,
+                        model_response=model_response,
+                        print_verbose=print_verbose,
+                        api_key=api_key,
+                        api_base=api_base,
+                        acompletion=acompletion,
+                        logging_obj=logging,
+                        optional_params=optional_params,
+                        litellm_params=litellm_params,
+                        logger_fn=logger_fn,
+                        timeout=timeout,  # type: ignore
+                        custom_prompt_dict=custom_prompt_dict,
+                        client=client,  # pass AsyncOpenAI, OpenAI client
+                        organization=organization,
+                        custom_llm_provider=custom_llm_provider,
+                    )
             except Exception as e:
                 ## LOGGING - log the original exception returned
                 logging.post_call(

litellm/proxy/_new_secret_config.yaml

@@ -6,7 +6,6 @@ model_list:
     api_base: https://exampleopenaiendpoint-production.up.railway.app/
 
 litellm_settings:
-  callbacks: ["otel"]
   cache: true
   cache_params:
     type: redis