mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(azure): key cached SDK clients on timeout too
The Azure client cache key read timeout as None for every call, since the value arrives in litellm_params. A client built for a short per-request timeout was then reused by later calls with the same credentials. Read timeout and max_retries from litellm_params into the key, through a typed adapter.
This commit is contained in:
parent
a43a6c3376
commit
e6bfb3c2ca
2 changed files with 87 additions and 7 deletions
|
|
@ -54,6 +54,14 @@ class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object):
|
|||
_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams)
|
||||
|
||||
|
||||
class _AzureClientLimits(TypedDict, total=False):
|
||||
max_retries: ReadOnly[object]
|
||||
timeout: ReadOnly[object]
|
||||
|
||||
|
||||
_AZURE_CLIENT_LIMITS_ADAPTER: Final = TypeAdapter(_AzureClientLimits)
|
||||
|
||||
|
||||
class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object):
|
||||
api_version: ReadOnly[str]
|
||||
base_url: ReadOnly[str]
|
||||
|
|
@ -568,10 +576,12 @@ class BaseAzureLLM(BaseOpenAILLM):
|
|||
client_initialization_params: Final[dict] = locals()
|
||||
client_initialization_params["is_async"] = _is_async
|
||||
_lp: Final = litellm_params or {}
|
||||
configured_max_retries: Final = _lp.get("max_retries")
|
||||
client_limits: Final = _AZURE_CLIENT_LIMITS_ADAPTER.validate_python(_lp)
|
||||
configured_max_retries: Final = client_limits.get("max_retries")
|
||||
client_initialization_params["max_retries"] = (
|
||||
DEFAULT_MAX_RETRIES if configured_max_retries is None else configured_max_retries
|
||||
)
|
||||
client_initialization_params["timeout"] = client_limits.get("timeout")
|
||||
_ad_provider: Final = _lp.get("azure_ad_token_provider")
|
||||
_ad_token: Final = _lp.get("azure_ad_token")
|
||||
_client_secret: Final = _lp.get("client_secret")
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from unittest.mock import ANY, MagicMock, patch
|
|||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
|
||||
|
||||
import litellm
|
||||
from litellm.llms.azure.common_utils import (
|
||||
|
|
@ -32,6 +33,17 @@ from litellm.types.secret_managers.get_azure_ad_token_provider import (
|
|||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
AzureSdkClient = AzureOpenAI | AsyncAzureOpenAI | OpenAI | AsyncOpenAI
|
||||
|
||||
|
||||
async def _send_chat_completion(client: AzureSdkClient | None, is_async: bool) -> None:
|
||||
assert client is not None
|
||||
response: Final = client.chat.completions.create(
|
||||
model="test-deployment", messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
if is_async:
|
||||
await response
|
||||
|
||||
|
||||
# Mock the necessary dependencies
|
||||
@pytest.fixture
|
||||
|
|
@ -2502,8 +2514,10 @@ async def test_azure_client_cache_respects_retry_limit(
|
|||
monkeypatch, is_async, api_version, first_retries
|
||||
):
|
||||
import httpx
|
||||
from openai import DEFAULT_MAX_RETRIES, RateLimitError
|
||||
from openai import RateLimitError
|
||||
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
from litellm.constants import DEFAULT_MAX_RETRIES
|
||||
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
requests = []
|
||||
|
|
@ -2545,11 +2559,7 @@ async def test_azure_client_cache_respects_retry_limit(
|
|||
**kwargs, litellm_params={"max_retries": DEFAULT_MAX_RETRIES}
|
||||
) is initial
|
||||
with pytest.raises(RateLimitError):
|
||||
response = no_retry.chat.completions.create(
|
||||
model="test-deployment", messages=[{"role": "user", "content": "hi"}]
|
||||
)
|
||||
if is_async:
|
||||
await response
|
||||
await _send_chat_completion(no_retry, is_async)
|
||||
assert len(requests) == 1
|
||||
assert no_retry is not initial
|
||||
assert (
|
||||
|
|
@ -2565,3 +2575,63 @@ async def test_azure_client_cache_respects_retry_limit(
|
|||
finally:
|
||||
sync_http.close()
|
||||
await async_http.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
@pytest.mark.parametrize("api_version", ["2024-02-01", "v1"])
|
||||
async def test_azure_client_cache_keeps_each_timeout_on_its_own_client(
|
||||
monkeypatch: pytest.MonkeyPatch, is_async: bool, api_version: str
|
||||
) -> None:
|
||||
from litellm.caching.llm_caching_handler import LLMClientCache
|
||||
|
||||
monkeypatch.setattr(litellm, "in_memory_llm_clients_cache", LLMClientCache())
|
||||
sent_read_timeouts: list[float | None] = []
|
||||
|
||||
def completes(request: httpx.Request) -> httpx.Response:
|
||||
sent_read_timeouts.append(request.extensions["timeout"]["read"])
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-timeout-cache",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "test-deployment",
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
sync_http: Final = httpx.Client(transport=httpx.MockTransport(completes))
|
||||
async_http: Final = httpx.AsyncClient(transport=httpx.MockTransport(completes))
|
||||
monkeypatch.setattr(litellm, "client_session", sync_http)
|
||||
monkeypatch.setattr(litellm, "aclient_session", async_http)
|
||||
llm: Final = BaseAzureLLM()
|
||||
|
||||
def client_for(litellm_params: dict[str, float]) -> AzureSdkClient | None:
|
||||
return llm.get_azure_openai_client(
|
||||
api_key="test-key",
|
||||
api_base="https://timeout-cache-test.openai.azure.com",
|
||||
api_version=api_version,
|
||||
_is_async=is_async,
|
||||
model="test-deployment",
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
try:
|
||||
short_client: Final = client_for({"timeout": 0.1})
|
||||
long_client: Final = client_for({"timeout": 30.0})
|
||||
default_client: Final = client_for({})
|
||||
await _send_chat_completion(short_client, is_async)
|
||||
await _send_chat_completion(long_client, is_async)
|
||||
|
||||
assert sent_read_timeouts == [0.1, 30.0]
|
||||
assert default_client is not short_client
|
||||
assert default_client is not long_client
|
||||
assert client_for({"timeout": 0.1}) is short_client
|
||||
assert client_for({"timeout": 30.0}) is long_client
|
||||
assert client_for({}) is default_client
|
||||
finally:
|
||||
sync_http.close()
|
||||
await async_http.aclose()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue