diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 45b104c493a..1d6baf60060 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -54,6 +54,14 @@ class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object): _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams) +class _AzureClientLimits(TypedDict, total=False): + max_retries: ReadOnly[object] + timeout: ReadOnly[object] + + +_AZURE_CLIENT_LIMITS_ADAPTER: Final = TypeAdapter(_AzureClientLimits) + + class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object): api_version: ReadOnly[str] base_url: ReadOnly[str] @@ -568,10 +576,12 @@ class BaseAzureLLM(BaseOpenAILLM): client_initialization_params: Final[dict] = locals() client_initialization_params["is_async"] = _is_async _lp: Final = litellm_params or {} - configured_max_retries: Final = _lp.get("max_retries") + client_limits: Final = _AZURE_CLIENT_LIMITS_ADAPTER.validate_python(_lp) + configured_max_retries: Final = client_limits.get("max_retries") client_initialization_params["max_retries"] = ( DEFAULT_MAX_RETRIES if configured_max_retries is None else configured_max_retries ) + client_initialization_params["timeout"] = client_limits.get("timeout") _ad_provider: Final = _lp.get("azure_ad_token_provider") _ad_token: Final = _lp.get("azure_ad_token") _client_secret: Final = _lp.get("client_secret") diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 6761c53e464..f565e08f79a 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -11,6 +11,7 @@ from unittest.mock import ANY, MagicMock, patch import httpx import pytest +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI import litellm from litellm.llms.azure.common_utils import ( @@ -32,6 +33,17 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import ( ) from litellm.types.utils import CallTypes +AzureSdkClient = AzureOpenAI | AsyncAzureOpenAI | OpenAI | AsyncOpenAI + + +async def _send_chat_completion(client: AzureSdkClient | None, is_async: bool) -> None: + assert client is not None + response: Final = client.chat.completions.create( + model="test-deployment", messages=[{"role": "user", "content": "hi"}] + ) + if is_async: + await response + # Mock the necessary dependencies @pytest.fixture @@ -2502,8 +2514,10 @@ async def test_azure_client_cache_respects_retry_limit( monkeypatch, is_async, api_version, first_retries ): import httpx - from openai import DEFAULT_MAX_RETRIES, RateLimitError + from openai import RateLimitError + from litellm.caching.llm_caching_handler import LLMClientCache + from litellm.constants import DEFAULT_MAX_RETRIES monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) requests = [] @@ -2545,11 +2559,7 @@ async def test_azure_client_cache_respects_retry_limit( **kwargs, litellm_params={"max_retries": DEFAULT_MAX_RETRIES} ) is initial with pytest.raises(RateLimitError): - response = no_retry.chat.completions.create( - model="test-deployment", messages=[{"role": "user", "content": "hi"}] - ) - if is_async: - await response + await _send_chat_completion(no_retry, is_async) assert len(requests) == 1 assert no_retry is not initial assert ( @@ -2565,3 +2575,63 @@ async def test_azure_client_cache_respects_retry_limit( finally: sync_http.close() await async_http.aclose() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("is_async", [False, True]) +@pytest.mark.parametrize("api_version", ["2024-02-01", "v1"]) +async def test_azure_client_cache_keeps_each_timeout_on_its_own_client( + monkeypatch: pytest.MonkeyPatch, is_async: bool, api_version: str +) -> None: + from litellm.caching.llm_caching_handler import LLMClientCache + + monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache()) + sent_read_timeouts: list[float | None] = [] + + def completes(request: httpx.Request) -> httpx.Response: + sent_read_timeouts.append(request.extensions["timeout"]["read"]) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-timeout-cache", + "object": "chat.completion", + "created": 0, + "model": "test-deployment", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"} + ], + }, + ) + + sync_http: Final = httpx.Client(transport=httpx.MockTransport(completes)) + async_http: Final = httpx.AsyncClient(transport=httpx.MockTransport(completes)) + monkeypatch.setattr(litellm, "client_session", sync_http) + monkeypatch.setattr(litellm, "aclient_session", async_http) + llm: Final = BaseAzureLLM() + + def client_for(litellm_params: dict[str, float]) -> AzureSdkClient | None: + return llm.get_azure_openai_client( + api_key="test-key", + api_base="https://timeout-cache-test.openai.azure.com", + api_version=api_version, + _is_async=is_async, + model="test-deployment", + litellm_params=litellm_params, + ) + + try: + short_client: Final = client_for({"timeout": 0.1}) + long_client: Final = client_for({"timeout": 30.0}) + default_client: Final = client_for({}) + await _send_chat_completion(short_client, is_async) + await _send_chat_completion(long_client, is_async) + + assert sent_read_timeouts == [0.1, 30.0] + assert default_client is not short_client + assert default_client is not long_client + assert client_for({"timeout": 0.1}) is short_client + assert client_for({"timeout": 30.0}) is long_client + assert client_for({}) is default_client + finally: + sync_http.close() + await async_http.aclose()