From dd1ccec7348b73f9a3d7d1ec8ac5fd29651de759 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Sat, 3 Jan 2026 15:50:53 -0800 Subject: [PATCH] refactor(utils): lazy load 15 additional imports to improve import time (#18613) * refactor(utils): lazy load 15 additional imports to improve import time - Move Rules, AsyncHTTPHandler, HTTPHandler to lazy loading via __getattr__ - Move get_num_retries_from_retry_policy, reset_retry_policy to lazy loading - Move get_secret to lazy loading - Move cached_imports functions (get_coroutine_checker, get_litellm_logging_class, get_set_callbacks) to lazy loading - Move core_helpers functions (get_litellm_metadata_from_kwargs, map_finish_reason, process_response_headers) to lazy loading - Move dot_notation_indexing functions (delete_nested_value, is_nested_path) to lazy loading - Move get_litellm_params functions to lazy loading - Move _ensure_extra_body_is_safe, get_formatted_prompt, get_response_headers, update_response_metadata to lazy loading - Move executor to lazy loading - Move BaseAnthropicMessagesConfig, BaseAudioTranscriptionConfig to lazy loading - Add type stubs in TYPE_CHECKING block for mypy type checking - These functions/classes are exported for other modules but not used internally in utils.py, so lazy loading is safe and improves startup performance * fix(utils): use getattr for Rules and get_coroutine_checker in client decorator - Update Rules() instantiation in client decorator to use getattr for lazy loading - Update Rules.has_pre_call_rules() usage in function_setup to use getattr - Update get_coroutine_checker() usage in client decorator to use getattr - Fixes NameError: name 'Rules' is not defined error that occurs when Rules is lazy-loaded * fix(utils): use getattr for get_litellm_logging_class in function_setup - Update get_litellm_logging_class() usage in function_setup to use getattr for lazy loading - Fixes NameError: name 'get_litellm_logging_class' is not defined error that occurs when get_litellm_logging_class is lazy-loaded * fix(utils): use getattr for get_set_callbacks in function_setup - Update get_set_callbacks() usage in function_setup to use getattr for lazy loading - Fixes NameError: name 'get_set_callbacks' is not defined error that occurs when get_set_callbacks is lazy-loaded * fix(utils): use getattr for all lazy-loaded imports in utils.py - Update update_response_metadata (4 occurrences) to use getattr - Update executor.submit (1 occurrence) to use getattr - Update get_num_retries_from_retry_policy (2 occurrences) to use getattr - Update reset_retry_policy (2 occurrences) to use getattr - Update is_nested_path and delete_nested_value (1 occurrence each) to use getattr - Update _ensure_extra_body_is_safe (1 occurrence) to use getattr Fixes NameError errors that occur when these functions/classes are lazy-loaded but used directly in utils.py * fix(utils): use getattr for _get_base_model_from_litellm_call_metadata in _get_base_model_from_metadata - Update _get_base_model_from_litellm_call_metadata usage to use getattr for lazy loading - Fixes NameError: name '_get_base_model_from_litellm_call_metadata' is not defined * fix(utils): use getattr for second _get_base_model_from_litellm_call_metadata usage - Fix the second occurrence of _get_base_model_from_litellm_call_metadata on line 7052 - Both occurrences in _get_base_model_from_metadata now use getattr for lazy loading * fix(utils): fix indentation in _get_base_model_from_metadata function * fix(utils): use getattr for get_litellm_metadata_from_kwargs in _get_litellm_params - Update get_litellm_metadata_from_kwargs usage to use getattr for lazy loading - Fixes NameError: name 'get_litellm_metadata_from_kwargs' is not defined * fix(utils): fix syntax error in get_litellm_metadata_from_kwargs fix - Move getattr call before cast statement to fix syntax error --- litellm/utils.py | 330 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 284 insertions(+), 46 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index d373b5102ef..6f4652c8278 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -72,39 +72,7 @@ from litellm.constants import ( TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) -# Import cached imports utilities -from litellm.litellm_core_utils.cached_imports import ( - get_coroutine_checker, - get_litellm_logging_class, - get_set_callbacks, -) -from litellm.litellm_core_utils.core_helpers import ( - get_litellm_metadata_from_kwargs, - map_finish_reason, - process_response_headers, -) -from litellm.litellm_core_utils.dot_notation_indexing import ( - delete_nested_value, - is_nested_path, -) -from litellm.litellm_core_utils.get_litellm_params import ( - _get_base_model_from_litellm_call_metadata, - get_litellm_params, -) -from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe -from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( - get_formatted_prompt, -) -from litellm.litellm_core_utils.llm_response_utils.get_headers import ( - get_response_headers, -) -from litellm.litellm_core_utils.rules import Rules -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler -from litellm.router_utils.get_retry_from_policy import ( - get_num_retries_from_retry_policy, - reset_retry_policy, -) -from litellm.secret_managers.main import get_secret + _CachingHandlerResponse = None _LLMCachingHandler = None @@ -280,16 +248,7 @@ from typing import ( from openai import OpenAIError as OriginalError -from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( - update_response_metadata, -) -from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.llms.base_llm.anthropic_messages.transformation import ( - BaseAnthropicMessagesConfig, -) -from litellm.llms.base_llm.audio_transcription.transformation import ( - BaseAudioTranscriptionConfig, -) +# These are lazy loaded via __getattr__ from litellm.llms.base_llm.base_utils import ( BaseLLMModelInfo, type_to_response_format_param, @@ -340,6 +299,49 @@ if TYPE_CHECKING: from litellm.llms.bedrock.common_utils import BedrockModelInfo from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.mistral.ocr.transformation import MistralOCRConfig + # Type stubs for lazy-loaded functions and classes + from litellm.litellm_core_utils.cached_imports import ( + get_coroutine_checker, + get_litellm_logging_class, + get_set_callbacks, + ) + from litellm.litellm_core_utils.core_helpers import ( + get_litellm_metadata_from_kwargs, + map_finish_reason, + process_response_headers, + ) + from litellm.litellm_core_utils.dot_notation_indexing import ( + delete_nested_value, + is_nested_path, + ) + from litellm.litellm_core_utils.get_litellm_params import ( + _get_base_model_from_litellm_call_metadata, + get_litellm_params, + ) + from litellm.litellm_core_utils.llm_request_utils import _ensure_extra_body_is_safe + from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( + get_formatted_prompt, + ) + from litellm.litellm_core_utils.llm_response_utils.get_headers import ( + get_response_headers, + ) + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata, + ) + from litellm.litellm_core_utils.rules import Rules + from litellm.litellm_core_utils.thread_pool_executor import executor + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig, + ) + from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig, + ) + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy, + reset_retry_policy, + ) + from litellm.secret_managers.main import get_secret from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig @@ -816,6 +818,7 @@ def function_setup( # noqa: PLR0915 + litellm.failure_callback ) ) + get_set_callbacks = getattr(sys.modules[__name__], 'get_set_callbacks') get_set_callbacks()(callback_list=callback_list, function_id=function_id) ## ASYNC CALLBACKS if len(litellm.input_callback) > 0: @@ -943,6 +946,7 @@ def function_setup( # noqa: PLR0915 elif kwargs.get("messages", None): messages = kwargs["messages"] ### PRE-CALL RULES ### + Rules = getattr(sys.modules[__name__], 'Rules') if ( Rules.has_pre_call_rules() and isinstance(messages, list) @@ -1075,6 +1079,7 @@ def function_setup( # noqa: PLR0915 call_type=call_type, ): stream = True + get_litellm_logging_class = getattr(sys.modules[__name__], 'get_litellm_logging_class') logging_obj = get_litellm_logging_class()( # Victim for object pool model=model, # type: ignore messages=messages, @@ -1158,6 +1163,8 @@ def _get_wrapper_num_retries( if num_retries is None: num_retries = litellm.num_retries if kwargs.get("retry_policy", None): + get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy') + reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy') retry_policy_num_retries = get_num_retries_from_retry_policy( exception=exception, retry_policy=kwargs.get("retry_policy"), @@ -1343,6 +1350,7 @@ def post_call_processing( def client(original_function): # noqa: PLR0915 + Rules = getattr(sys.modules[__name__], 'Rules') rules_obj = Rules() @wraps(original_function) @@ -1528,6 +1536,7 @@ def client(original_function): # noqa: PLR0915 ) else: # RETURN RESULT + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1571,6 +1580,7 @@ def client(original_function): # noqa: PLR0915 # Copy the current context to propagate it to the background thread # This is essential for OpenTelemetry span context propagation ctx = contextvars.copy_context() + executor = getattr(sys.modules[__name__], 'executor') executor.submit( ctx.run, logging_obj.success_handler, @@ -1579,6 +1589,7 @@ def client(original_function): # noqa: PLR0915 end_time, ) # RETURN RESULT + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1595,6 +1606,8 @@ def client(original_function): # noqa: PLR0915 kwargs.get("num_retries", None) or litellm.num_retries or None ) if kwargs.get("retry_policy", None): + get_num_retries_from_retry_policy = getattr(sys.modules[__name__], 'get_num_retries_from_retry_policy') + reset_retry_policy = getattr(sys.modules[__name__], 'reset_retry_policy') num_retries = get_num_retries_from_retry_policy( exception=e, retry_policy=kwargs.get("retry_policy"), @@ -1766,6 +1779,7 @@ def client(original_function): # noqa: PLR0915 chunks, messages=kwargs.get("messages", None) ) else: + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1830,6 +1844,7 @@ def client(original_function): # noqa: PLR0915 end_time=end_time, ) + update_response_metadata = getattr(sys.modules[__name__], 'update_response_metadata') update_response_metadata( result=result, logging_obj=logging_obj, @@ -1905,6 +1920,7 @@ def client(original_function): # noqa: PLR0915 setattr(e, "timeout", timeout) raise e + get_coroutine_checker = getattr(sys.modules[__name__], 'get_coroutine_checker') is_coroutine = get_coroutine_checker().is_async_callable(original_function) # Return the appropriate wrapper based on the original function type @@ -4448,6 +4464,8 @@ def get_optional_params( # noqa: PLR0915 # Apply nested drops from additional_drop_params if additional_drop_params: + is_nested_path = getattr(sys.modules[__name__], 'is_nested_path') + delete_nested_value = getattr(sys.modules[__name__], 'delete_nested_value') nested_paths = [p for p in additional_drop_params if is_nested_path(p)] for path in nested_paths: optional_params = delete_nested_value(optional_params, path) @@ -4497,6 +4515,7 @@ def add_provider_specific_params_to_optional_params( else: processed_extra_body = initial_extra_body + _ensure_extra_body_is_safe = getattr(sys.modules[__name__], '_ensure_extra_body_is_safe') optional_params["extra_body"] = _ensure_extra_body_is_safe( extra_body=processed_extra_body ) @@ -7022,14 +7041,14 @@ def _get_base_model_from_metadata(model_call_details=None): return _base_model metadata = litellm_params.get("metadata", {}) - base_model_from_metadata = _get_base_model_from_litellm_call_metadata( - metadata=metadata - ) + _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata') + base_model_from_metadata = _get_base_model_from_litellm_call_metadata(metadata=metadata) if base_model_from_metadata is not None: return base_model_from_metadata # Also check litellm_metadata (used by Responses API and other generic API calls) litellm_metadata = litellm_params.get("litellm_metadata", {}) + _get_base_model_from_litellm_call_metadata = getattr(sys.modules[__name__], '_get_base_model_from_litellm_call_metadata') return _get_base_model_from_litellm_call_metadata(metadata=litellm_metadata) return None @@ -8433,6 +8452,7 @@ def get_end_user_id_for_cost_tracking( service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking. """ + get_litellm_metadata_from_kwargs = getattr(sys.modules[__name__], 'get_litellm_metadata_from_kwargs') _metadata = cast( dict, get_litellm_metadata_from_kwargs(dict(litellm_params=litellm_params)) ) @@ -8988,4 +9008,222 @@ def __getattr__(name: str) -> Any: # noqa: PLR0915 _globals["MistralOCRConfig"] = _MistralOCRConfig return _globals["MistralOCRConfig"] + # Lazy load Rules to avoid loading at module import time + if name == "Rules": + # Check if already cached + if "Rules" not in _globals: + from litellm.litellm_core_utils.rules import Rules as _Rules + _globals["Rules"] = _Rules + return _globals["Rules"] + + # Lazy load AsyncHTTPHandler and HTTPHandler to avoid loading at module import time + if name == "AsyncHTTPHandler": + # Check if already cached + if "AsyncHTTPHandler" not in _globals: + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler as _AsyncHTTPHandler, + ) + _globals["AsyncHTTPHandler"] = _AsyncHTTPHandler + return _globals["AsyncHTTPHandler"] + + if name == "HTTPHandler": + # Check if already cached + if "HTTPHandler" not in _globals: + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler as _HTTPHandler, + ) + _globals["HTTPHandler"] = _HTTPHandler + return _globals["HTTPHandler"] + + # Lazy load get_num_retries_from_retry_policy and reset_retry_policy to avoid loading at module import time + if name == "get_num_retries_from_retry_policy": + # Check if already cached + if "get_num_retries_from_retry_policy" not in _globals: + from litellm.router_utils.get_retry_from_policy import ( + get_num_retries_from_retry_policy as _get_num_retries_from_retry_policy, + ) + _globals["get_num_retries_from_retry_policy"] = _get_num_retries_from_retry_policy + return _globals["get_num_retries_from_retry_policy"] + + if name == "reset_retry_policy": + # Check if already cached + if "reset_retry_policy" not in _globals: + from litellm.router_utils.get_retry_from_policy import ( + reset_retry_policy as _reset_retry_policy, + ) + _globals["reset_retry_policy"] = _reset_retry_policy + return _globals["reset_retry_policy"] + + # Lazy load get_secret to avoid loading at module import time + if name == "get_secret": + # Check if already cached + if "get_secret" not in _globals: + from litellm.secret_managers.main import get_secret as _get_secret + _globals["get_secret"] = _get_secret + return _globals["get_secret"] + + # Lazy load cached_imports functions to avoid loading at module import time + if name == "get_coroutine_checker": + # Check if already cached + if "get_coroutine_checker" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_coroutine_checker as _get_coroutine_checker, + ) + _globals["get_coroutine_checker"] = _get_coroutine_checker + return _globals["get_coroutine_checker"] + + if name == "get_litellm_logging_class": + # Check if already cached + if "get_litellm_logging_class" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_litellm_logging_class as _get_litellm_logging_class, + ) + _globals["get_litellm_logging_class"] = _get_litellm_logging_class + return _globals["get_litellm_logging_class"] + + if name == "get_set_callbacks": + # Check if already cached + if "get_set_callbacks" not in _globals: + from litellm.litellm_core_utils.cached_imports import ( + get_set_callbacks as _get_set_callbacks, + ) + _globals["get_set_callbacks"] = _get_set_callbacks + return _globals["get_set_callbacks"] + + # Lazy load core_helpers functions to avoid loading at module import time + if name == "get_litellm_metadata_from_kwargs": + # Check if already cached + if "get_litellm_metadata_from_kwargs" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + get_litellm_metadata_from_kwargs as _get_litellm_metadata_from_kwargs, + ) + _globals["get_litellm_metadata_from_kwargs"] = _get_litellm_metadata_from_kwargs + return _globals["get_litellm_metadata_from_kwargs"] + + if name == "map_finish_reason": + # Check if already cached + if "map_finish_reason" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + map_finish_reason as _map_finish_reason, + ) + _globals["map_finish_reason"] = _map_finish_reason + return _globals["map_finish_reason"] + + if name == "process_response_headers": + # Check if already cached + if "process_response_headers" not in _globals: + from litellm.litellm_core_utils.core_helpers import ( + process_response_headers as _process_response_headers, + ) + _globals["process_response_headers"] = _process_response_headers + return _globals["process_response_headers"] + + # Lazy load dot_notation_indexing functions to avoid loading at module import time + if name == "delete_nested_value": + # Check if already cached + if "delete_nested_value" not in _globals: + from litellm.litellm_core_utils.dot_notation_indexing import ( + delete_nested_value as _delete_nested_value, + ) + _globals["delete_nested_value"] = _delete_nested_value + return _globals["delete_nested_value"] + + if name == "is_nested_path": + # Check if already cached + if "is_nested_path" not in _globals: + from litellm.litellm_core_utils.dot_notation_indexing import ( + is_nested_path as _is_nested_path, + ) + _globals["is_nested_path"] = _is_nested_path + return _globals["is_nested_path"] + + # Lazy load get_litellm_params functions to avoid loading at module import time + if name == "_get_base_model_from_litellm_call_metadata": + # Check if already cached + if "_get_base_model_from_litellm_call_metadata" not in _globals: + from litellm.litellm_core_utils.get_litellm_params import ( + _get_base_model_from_litellm_call_metadata as __get_base_model_from_litellm_call_metadata, + ) + _globals["_get_base_model_from_litellm_call_metadata"] = __get_base_model_from_litellm_call_metadata + return _globals["_get_base_model_from_litellm_call_metadata"] + + if name == "get_litellm_params": + # Check if already cached + if "get_litellm_params" not in _globals: + from litellm.litellm_core_utils.get_litellm_params import ( + get_litellm_params as _get_litellm_params, + ) + _globals["get_litellm_params"] = _get_litellm_params + return _globals["get_litellm_params"] + + # Lazy load _ensure_extra_body_is_safe to avoid loading at module import time + if name == "_ensure_extra_body_is_safe": + # Check if already cached + if "_ensure_extra_body_is_safe" not in _globals: + from litellm.litellm_core_utils.llm_request_utils import ( + _ensure_extra_body_is_safe as __ensure_extra_body_is_safe, + ) + _globals["_ensure_extra_body_is_safe"] = __ensure_extra_body_is_safe + return _globals["_ensure_extra_body_is_safe"] + + # Lazy load get_formatted_prompt to avoid loading at module import time + if name == "get_formatted_prompt": + # Check if already cached + if "get_formatted_prompt" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_formatted_prompt import ( + get_formatted_prompt as _get_formatted_prompt, + ) + _globals["get_formatted_prompt"] = _get_formatted_prompt + return _globals["get_formatted_prompt"] + + # Lazy load get_response_headers to avoid loading at module import time + if name == "get_response_headers": + # Check if already cached + if "get_response_headers" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.get_headers import ( + get_response_headers as _get_response_headers, + ) + _globals["get_response_headers"] = _get_response_headers + return _globals["get_response_headers"] + + # Lazy load update_response_metadata to avoid loading at module import time + if name == "update_response_metadata": + # Check if already cached + if "update_response_metadata" not in _globals: + from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( + update_response_metadata as _update_response_metadata, + ) + _globals["update_response_metadata"] = _update_response_metadata + return _globals["update_response_metadata"] + + # Lazy load executor to avoid loading at module import time + if name == "executor": + # Check if already cached + if "executor" not in _globals: + from litellm.litellm_core_utils.thread_pool_executor import ( + executor as _executor, + ) + _globals["executor"] = _executor + return _globals["executor"] + + # Lazy load BaseAnthropicMessagesConfig to avoid loading at module import time + if name == "BaseAnthropicMessagesConfig": + # Check if already cached + if "BaseAnthropicMessagesConfig" not in _globals: + from litellm.llms.base_llm.anthropic_messages.transformation import ( + BaseAnthropicMessagesConfig as _BaseAnthropicMessagesConfig, + ) + _globals["BaseAnthropicMessagesConfig"] = _BaseAnthropicMessagesConfig + return _globals["BaseAnthropicMessagesConfig"] + + # Lazy load BaseAudioTranscriptionConfig to avoid loading at module import time + if name == "BaseAudioTranscriptionConfig": + # Check if already cached + if "BaseAudioTranscriptionConfig" not in _globals: + from litellm.llms.base_llm.audio_transcription.transformation import ( + BaseAudioTranscriptionConfig as _BaseAudioTranscriptionConfig, + ) + _globals["BaseAudioTranscriptionConfig"] = _BaseAudioTranscriptionConfig + return _globals["BaseAudioTranscriptionConfig"] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}")