From 46cb9295daa1e5f2d93acc75919fe85d886cdd3f Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Fri, 13 Feb 2026 15:06:15 -0800 Subject: [PATCH] perf: optimize get_openai_client_cache_key and pre-compute SDK init params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace 16-param loop with 6-param tuple iteration driven by _CACHE_KEY_IDENTITY_PARAMS (single source of truth) - Pre-compute OpenAI/AzureOpenAI __init__ params at module load instead of calling inspect.signature() on every request - Add 3 safety-net tests to detect cache key param drift Line profile: 14.8µs → 6.6µs per call (2.25x faster) --- litellm/llms/openai/common_utils.py | 75 +++++------ .../llms/openai/test_openai_common_utils.py | 123 +++++++++++++++++- 2 files changed, 156 insertions(+), 42 deletions(-) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index ce470f04aca..9e677432763 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -14,6 +14,8 @@ from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI if TYPE_CHECKING: from aiohttp import ClientSession +import inspect + import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import ( @@ -23,6 +25,29 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def _get_client_init_params(cls: type) -> List[str]: + """Extract __init__ parameter names (excluding 'self') from a class.""" + return [p for p in inspect.signature(cls.__init__).parameters if p != "self"] + + +_OPENAI_INIT_PARAMS: List[str] = _get_client_init_params(OpenAI) +_AZURE_OPENAI_INIT_PARAMS: List[str] = _get_client_init_params(AzureOpenAI) + +# Ordered params included in the cache key (excluding api_key which is hashed separately). +# get_openai_client_cache_key iterates this tuple; safety-net tests use it to detect drift. +_CACHE_KEY_IDENTITY_PARAMS = ( + "is_async", + "api_base", + "api_version", + "timeout", + "max_retries", + "organization", +) + +# Full set of params that affect client identity (includes api_key). +_CACHE_KEY_PARAMS = frozenset(_CACHE_KEY_IDENTITY_PARAMS) | {"api_key"} + + class OpenAIError(BaseLLMException): def __init__( self, @@ -145,56 +170,24 @@ class BaseOpenAILLM: client_initialization_params: dict, client_type: Literal["openai", "azure"] ) -> str: """Creates a cache key for the OpenAI client based on the client initialization parameters""" - hashed_api_key = None - if client_initialization_params.get("api_key") is not None: - hash_object = hashlib.sha256( - client_initialization_params.get("api_key", "").encode() - ) - # Hexadecimal representation of the hash - hashed_api_key = hash_object.hexdigest() - - # Create a more readable cache key using a list of key-value pairs - key_parts = [ - f"hashed_api_key={hashed_api_key}", - f"is_async={client_initialization_params.get('is_async')}", - ] - - LITELLM_CLIENT_SPECIFIC_PARAMS = [ - "timeout", - "max_retries", - "organization", - "api_base", - ] - openai_client_fields = ( - BaseOpenAILLM.get_openai_client_initialization_param_fields( - client_type=client_type - ) - + LITELLM_CLIENT_SPECIFIC_PARAMS + api_key = client_initialization_params.get("api_key") + hashed_api_key = ( + hashlib.sha256(api_key.encode()).hexdigest() if api_key else None + ) + return ( + f"{client_type},{hashed_api_key}," + + ",".join(str(client_initialization_params.get(p)) for p in _CACHE_KEY_IDENTITY_PARAMS) ) - - for param in openai_client_fields: - key_parts.append(f"{param}={client_initialization_params.get(param)}") - - _cache_key = ",".join(key_parts) - return _cache_key @staticmethod def get_openai_client_initialization_param_fields( client_type: Literal["openai", "azure"] ) -> List[str]: """Returns a list of fields that are used to initialize the OpenAI client""" - import inspect - - from openai import AzureOpenAI, OpenAI - if client_type == "openai": - signature = inspect.signature(OpenAI.__init__) + return _OPENAI_INIT_PARAMS else: - signature = inspect.signature(AzureOpenAI.__init__) - - # Extract parameter names, excluding 'self' - param_names = [param for param in signature.parameters if param != "self"] - return param_names + return _AZURE_OPENAI_INIT_PARAMS @staticmethod def _get_async_http_client( diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py index 469005d103f..a05794f4ebe 100644 --- a/tests/test_litellm/llms/openai/test_openai_common_utils.py +++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py @@ -1,3 +1,4 @@ +import inspect import os import sys from unittest.mock import MagicMock, call, patch @@ -9,7 +10,7 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm -from litellm.llms.openai.common_utils import BaseOpenAILLM +from litellm.llms.openai.common_utils import BaseOpenAILLM, _CACHE_KEY_PARAMS # Test parameters for different API functions API_FUNCTION_PARAMS = [ @@ -129,3 +130,123 @@ async def test_openai_client_reuse(function_name, is_async, args): # Verify we tried to get from cache 10 times (once per request) assert mock_get_cache.call_count == 10, "Should check cache for each request" + + +def test_precomputed_init_params_match_inspect_signature(): + """ + Verify that the pre-computed _OPENAI_INIT_PARAMS and _AZURE_OPENAI_INIT_PARAMS + match what inspect.signature() returns. If the OpenAI SDK changes its __init__ + params, this test will fail — signaling the constants need updating. + """ + import inspect + + from openai import AzureOpenAI, OpenAI + + from litellm.llms.openai.common_utils import ( + _AZURE_OPENAI_INIT_PARAMS, + _OPENAI_INIT_PARAMS, + ) + + expected_openai = [ + p for p in inspect.signature(OpenAI.__init__).parameters if p != "self" + ] + expected_azure = [ + p for p in inspect.signature(AzureOpenAI.__init__).parameters if p != "self" + ] + + assert _OPENAI_INIT_PARAMS == expected_openai + assert _AZURE_OPENAI_INIT_PARAMS == expected_azure + + +@pytest.mark.parametrize("client_type", ["openai", "azure"]) +def test_get_openai_client_initialization_param_fields(client_type): + """Verify the method returns the correct pre-computed params for each client type.""" + result = BaseOpenAILLM.get_openai_client_initialization_param_fields(client_type) + assert isinstance(result, list) + assert len(result) > 0 + assert "self" not in result + + +# --- Safety-net tests for cache key coverage --- +# _CACHE_KEY_PARAMS is imported from common_utils (single source of truth) + +# _get_openai_client params that don't affect client identity +_IGNORED_OPENAI = {"self", "client", "shared_session"} + +# get_azure_openai_client params that don't affect client identity +# _is_async maps to is_async in the cache key dict +_IGNORED_AZURE = {"self", "client", "litellm_params", "model"} +_AZURE_CACHE_KEY_PARAMS = _CACHE_KEY_PARAMS | {"_is_async"} + + +def test_openai_cache_key_covers_all_params(): + """ + If someone adds a new param to _get_openai_client that affects client + identity, this test will fail — signaling the cache key needs updating. + """ + from litellm.llms.openai.openai import OpenAIChatCompletion + + sig_params = set( + inspect.signature(OpenAIChatCompletion._get_openai_client).parameters.keys() + ) + covered = _CACHE_KEY_PARAMS | _IGNORED_OPENAI + uncovered = sig_params - covered + assert uncovered == set(), ( + f"_get_openai_client has new param(s) {uncovered} not in cache key or ignore list. " + f"Update get_openai_client_cache_key or add to _IGNORED_OPENAI." + ) + + +def test_azure_cache_key_covers_all_params(): + """ + If someone adds a new param to get_azure_openai_client that affects client + identity, this test will fail — signaling the cache key needs updating. + """ + from litellm.llms.azure.common_utils import BaseAzureLLM + + sig_params = set( + inspect.signature(BaseAzureLLM.get_azure_openai_client).parameters.keys() + ) + covered = _AZURE_CACHE_KEY_PARAMS | _IGNORED_AZURE + uncovered = sig_params - covered + assert uncovered == set(), ( + f"get_azure_openai_client has new param(s) {uncovered} not in cache key or ignore list. " + f"Update get_openai_client_cache_key or add to _IGNORED_AZURE." + ) + + +def test_cache_key_format(): + """ + Verify cache key contains all expected components and that different + api_version values produce different keys (Azure collision regression). + """ + import hashlib + + params = { + "api_key": "sk-test-key-123", + "is_async": True, + "api_base": "https://api.openai.com/v1", + "api_version": "2024-02-01", + "timeout": 600, + "max_retries": 2, + "organization": "org-abc", + } + + key = BaseOpenAILLM.get_openai_client_cache_key(params, "openai") + + expected_hash = hashlib.sha256(b"sk-test-key-123").hexdigest() + assert "openai" in key + assert expected_hash in key + assert "True" in key # is_async + assert "https://api.openai.com/v1" in key + assert "2024-02-01" in key + assert "600" in key + assert "2" in key # max_retries + assert "org-abc" in key + + # Azure collision regression: different api_version must produce different keys + params_v1 = {**params, "api_version": "2024-02-01"} + params_v2 = {**params, "api_version": "2024-06-01"} + key_v1 = BaseOpenAILLM.get_openai_client_cache_key(params_v1, "azure") + key_v2 = BaseOpenAILLM.get_openai_client_cache_key(params_v2, "azure") + assert key_v1 != key_v2, "Different api_version values must produce different cache keys"