mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-27 01:22:18 +00:00
f6882246d4
* ci: run the unit_selection.sh shard files on every event instead of only fork pull requests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: rename fork-flag to unit-flag now that it applies on every event * test: move tests/test_litellm root and small trees into tests/unit Pure renames, no content changes. Follow-up commits in this PR fix references, merge the three files that already existed in tests/unit, keep live-provider tests in tests/test_litellm and wire CI. * test: carry tests/test_litellm conftest isolation into tests/unit Callback lists, routing fallbacks, cached HTTP clients, logger state, AWS, proxy-URL and keychain env, and session-end client cleanup now reset for unit tests too. The environment isolation owns its MonkeyPatch so a test's own monkeypatch is undone before the model-cost teardown runs. * test: merge, split and prune the moved root and small-tree tests Merge batches/test_batch_utils.py and the chat_completions and messages dispatch tests into the files that already existed in tests/unit. Keep the live Gemini interactions tests, the async image-fetch format test and the OpenAI embedding scorer test in tests/test_litellm since they need real network or keys. Put test_router.py under tests/unit/test_router so the existing package no longer shadows it. Delete eight tests the audit found superseded by stronger ones kept in this move. * ci: run the moved root and small-tree tests under their legacy flags Add the misc and responses-caching-types flags to unit_selection.sh and CircleCI, extend enterprise-routing and mcp-integration, and point the legacy GHA shards, Makefile, redis-compat workflow, merge smoke manifest and change classifier at the new paths. * test: make the new tests/unit directories packages tests/unit/test_package_layout.py requires every directory to carry an __init__.py, and without one the moved and retained test_litellm_responses_bridge.py modules collide on import. * test: scope the unit socket block to tests/unit in shared sessions The GHA shards collect the legacy test-path and the unit selection in one pytest session. The unit conftest's loopback-only block leaked into legacy modules that reach the network at import. The legacy conftest now lifts the restriction at collect and setup time, and the unit conftest re-applies it when collecting its own modules. * test: give the shard-script tests their own GITHUB_OUTPUT They only passed where the runner set it. The CircleCI unit job's env allowlist drops it, so the script's redirect failed there. * test: point the router and module-deletion checks at tests/unit router_code_coverage and code_qa_check_tests only searched tests/test_litellm, so the moved router tests no longer counted. The two silent-experiment tests the audit deleted were the only direct callers of those methods; they are replaced with tests that assert the forwarded shadow request and the recursion guard. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
779 lines
31 KiB
Python
779 lines
31 KiB
Python
"""
|
|
Unit tests for per-deployment num_retries in litellm_params
|
|
GitHub Issue: #18968 - Per-deployment max_retries/num_retries in litellm_params is not used in retry logic
|
|
"""
|
|
|
|
import httpx
|
|
import pytest
|
|
import pytest_asyncio
|
|
from unittest.mock import patch
|
|
|
|
import litellm
|
|
from litellm import Router
|
|
from litellm.types.router import RetryPolicy
|
|
from litellm.integrations.custom_logger import CustomLogger
|
|
|
|
|
|
class TestPerDeploymentNumRetries:
|
|
"""Test that per-deployment num_retries in litellm_params is correctly used."""
|
|
|
|
def test_set_deployment_num_retries_on_exception(self):
|
|
"""
|
|
Test that _set_deployment_num_retries_on_exception sets num_retries
|
|
on the exception from the deployment's litellm_params.
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
"num_retries": 5, # Per-deployment setting
|
|
},
|
|
},
|
|
],
|
|
num_retries=1, # Global setting
|
|
)
|
|
|
|
deployment = router.model_list[0]
|
|
|
|
# Create a mock exception without num_retries
|
|
class MockException(Exception):
|
|
pass
|
|
|
|
exc = MockException("test error")
|
|
assert not hasattr(exc, "num_retries") or exc.num_retries is None
|
|
|
|
# Call the helper
|
|
router._set_deployment_num_retries_on_exception(exc, deployment)
|
|
|
|
# Verify num_retries was set from deployment
|
|
assert exc.num_retries == 5
|
|
|
|
def test_set_deployment_num_retries_does_not_override_existing(self):
|
|
"""
|
|
Test that _set_deployment_num_retries_on_exception does NOT override
|
|
if exception already has num_retries set.
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
"num_retries": 5,
|
|
},
|
|
},
|
|
],
|
|
num_retries=1,
|
|
)
|
|
|
|
deployment = router.model_list[0]
|
|
|
|
# Create an exception that already has num_retries
|
|
class MockException(Exception):
|
|
num_retries = 10 # Already set
|
|
|
|
exc = MockException("test error")
|
|
|
|
# Call the helper
|
|
router._set_deployment_num_retries_on_exception(exc, deployment)
|
|
|
|
# Verify num_retries was NOT overridden
|
|
assert exc.num_retries == 10
|
|
|
|
def test_deployment_without_num_retries(self):
|
|
"""
|
|
Test that _set_deployment_num_retries_on_exception does nothing
|
|
if deployment has no num_retries set.
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
# No num_retries set
|
|
},
|
|
},
|
|
],
|
|
num_retries=3,
|
|
)
|
|
|
|
deployment = router.model_list[0]
|
|
|
|
class MockException(Exception):
|
|
pass
|
|
|
|
exc = MockException("test error")
|
|
|
|
# Call the helper
|
|
router._set_deployment_num_retries_on_exception(exc, deployment)
|
|
|
|
# Verify num_retries was not set (deployment has no num_retries)
|
|
assert not hasattr(exc, "num_retries") or exc.num_retries is None
|
|
|
|
def test_request_level_num_retries_takes_precedence(self):
|
|
"""
|
|
Test that request-level num_retries (passed in kwargs) is still respected.
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
"num_retries": 5,
|
|
},
|
|
},
|
|
],
|
|
num_retries=1,
|
|
)
|
|
|
|
# Pass num_retries in request kwargs - this should take precedence
|
|
kwargs = {"num_retries": 10}
|
|
router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs)
|
|
assert kwargs["num_retries"] == 10 # Request-level takes precedence
|
|
|
|
def test_update_kwargs_does_not_fill_in_a_num_retries_default(self):
|
|
"""
|
|
A request that carries no num_retries must stay that way through
|
|
_update_kwargs_before_fallbacks. Filling in the router default here is what made a
|
|
request indistinguishable from "no request value", which let a deployment's
|
|
litellm_params.num_retries outrank the header/body value.
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
},
|
|
},
|
|
],
|
|
num_retries=7, # Global setting
|
|
)
|
|
|
|
kwargs: dict = {}
|
|
router._update_kwargs_before_fallbacks(model="test-model", kwargs=kwargs)
|
|
assert kwargs.get("num_retries") is None
|
|
|
|
def test_set_deployment_num_retries_with_string_value(self):
|
|
"""
|
|
Test that _set_deployment_num_retries_on_exception handles string values
|
|
from environment variables correctly.
|
|
GitHub Issue: #19481
|
|
"""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "test-model",
|
|
"litellm_params": {
|
|
"model": "openai/gpt-4",
|
|
"api_key": "test-key",
|
|
"num_retries": "6", # String value (as from env var)
|
|
},
|
|
},
|
|
],
|
|
num_retries=0, # Global setting
|
|
)
|
|
|
|
deployment = router.model_list[0]
|
|
|
|
class MockException(Exception):
|
|
pass
|
|
|
|
exc = MockException("test error")
|
|
|
|
# Call the helper
|
|
router._set_deployment_num_retries_on_exception(exc, deployment)
|
|
|
|
# Verify num_retries was converted from string to int
|
|
assert exc.num_retries == 6
|
|
|
|
|
|
class TestNumRetriesNoneGuard:
|
|
"""
|
|
Regression tests for the num_retries=None TypeError in async_function_with_retries.
|
|
|
|
When num_retries reaches async_function_with_retries as None - e.g. a caller passes
|
|
num_retries=None explicitly (dict.get() does not fall back on an existing None value),
|
|
an auto_router/complexity_router path does not propagate it, or
|
|
Router.update_settings(num_retries=None) is used - AND the underlying call fails with a
|
|
retryable error, the comparison `if num_retries > 0:` raised:
|
|
|
|
TypeError: '>' not supported between instances of 'NoneType' and 'int'
|
|
|
|
This masked the real upstream error (rate limit / connection / 5xx) behind a TypeError.
|
|
Related issues: #23316, #25889, #23699, #28126.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _mock_router(num_retries=2):
|
|
return Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "mock-model",
|
|
"litellm_params": {
|
|
"model": "gpt-4o-mini",
|
|
"mock_response": "ok",
|
|
},
|
|
}
|
|
],
|
|
num_retries=num_retries,
|
|
)
|
|
|
|
def test_update_kwargs_preserves_an_explicit_zero(self):
|
|
"""
|
|
An explicit num_retries=0 must survive _update_kwargs_before_fallbacks (retries stay
|
|
disabled), and an explicit None must not be turned into a value that reads as a
|
|
request-level setting. Resolving None to the router default is
|
|
async_function_with_retries' job, which the behavioural tests below pin.
|
|
"""
|
|
router = self._mock_router(num_retries=4)
|
|
|
|
kwargs: dict = {"num_retries": 0}
|
|
router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs)
|
|
assert kwargs["num_retries"] == 0
|
|
|
|
kwargs = {"num_retries": None}
|
|
router._update_kwargs_before_fallbacks(model="mock-model", kwargs=kwargs)
|
|
assert kwargs.get("num_retries") is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_acompletion_num_retries_none_does_not_raise_typeerror(self):
|
|
"""
|
|
Per-request num_retries=None + a retryable error must NOT raise TypeError.
|
|
The router falls back to its configured num_retries and retries the (transient)
|
|
error, so the request succeeds.
|
|
"""
|
|
router = self._mock_router(num_retries=2)
|
|
with patch("asyncio.sleep", return_value=None):
|
|
response = await router.acompletion(
|
|
model="mock-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
num_retries=None, # the trigger
|
|
mock_testing_rate_limit_error=True, # retryable error path
|
|
)
|
|
assert response.choices[0].message.content == "ok"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_function_with_retries_none_falls_back_to_zero(self):
|
|
"""
|
|
When both the per-request value AND the router-level setting are None
|
|
(e.g. after Router.update_settings(num_retries=None), #28126), num_retries must
|
|
fall back to 0 and the real retryable error must surface - not a TypeError.
|
|
"""
|
|
router = self._mock_router(num_retries=0)
|
|
router.num_retries = None # simulate update_settings(num_retries=None)
|
|
|
|
async def failing_fn(*args, **kwargs):
|
|
raise litellm.RateLimitError(
|
|
message="boom", model="mock-model", llm_provider="openai"
|
|
)
|
|
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.RateLimitError):
|
|
await router.async_function_with_retries(
|
|
original_function=failing_fn,
|
|
model="mock-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
num_retries=None,
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_function_with_retries_none_falls_back_to_router_default(self):
|
|
"""
|
|
A None per-request num_retries falls back to the router-level setting, so retries
|
|
still happen (original_function is invoked more than once) before the real error
|
|
is raised - proving None did not silently disable retries or crash.
|
|
"""
|
|
router = self._mock_router(num_retries=3)
|
|
calls = {"n": 0}
|
|
|
|
async def failing_fn(*args, **kwargs):
|
|
calls["n"] += 1
|
|
raise litellm.InternalServerError(
|
|
message="boom", model="mock-model", llm_provider="openai"
|
|
)
|
|
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.async_function_with_retries(
|
|
original_function=failing_fn,
|
|
model="mock-model",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
metadata={}, # populated by acompletion in the real path; log_retry needs it
|
|
num_retries=None,
|
|
)
|
|
|
|
# 1 initial attempt + at least 1 retry -> proves None fell back to a positive int
|
|
assert calls["n"] >= 2
|
|
|
|
|
|
class TestNoProviderRetryAmplification:
|
|
"""
|
|
A routed request must reach the upstream provider exactly ``1 + <router retries>``
|
|
times. The Router is the sole retry owner for routed calls, so the provider SDK
|
|
must never retry on top of it. Otherwise a per-deployment ``num_retries`` set in
|
|
``litellm_params`` is applied twice - once by the Router loop and once as the
|
|
provider client's ``max_retries`` - turning one request into ``(1 + num_retries) ** 2``
|
|
upstream requests.
|
|
|
|
These tests count actual upstream HTTP requests through the full Router completion
|
|
path by injecting a counting transport via ``litellm.aclient_session`` (the
|
|
documented seam the OpenAI client builder reads), so both Router-level and any
|
|
provider-SDK-level retries are observed.
|
|
"""
|
|
|
|
@staticmethod
|
|
def _install_counting_upstream() -> dict:
|
|
"""Route every upstream POST to a 500 and count it. ``retry-after: 0`` keeps
|
|
provider-SDK backoff at zero so a mutated (double-retrying) build stays fast."""
|
|
counter = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
counter["n"] += 1
|
|
return httpx.Response(
|
|
500,
|
|
headers={"retry-after": "0"},
|
|
json={"error": {"message": "boom", "type": "server_error"}},
|
|
)
|
|
|
|
litellm.aclient_session = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
return counter
|
|
|
|
@pytest_asyncio.fixture(autouse=True)
|
|
async def _isolate_clients(self):
|
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
|
yield
|
|
session = litellm.aclient_session
|
|
litellm.aclient_session = None
|
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
|
if session is not None:
|
|
await session.aclose()
|
|
|
|
@staticmethod
|
|
def _router(api_base: str, litellm_params: dict, **router_kwargs) -> Router:
|
|
params = {"model": "openai/gpt-4o-mini", "api_base": api_base, "api_key": "sk-fake"}
|
|
params.update(litellm_params)
|
|
return Router(model_list=[{"model_name": "mock", "litellm_params": params}], **router_kwargs)
|
|
|
|
async def _call_and_count(self, router: Router, **call_kwargs) -> int:
|
|
counter = self._install_counting_upstream()
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.acompletion(
|
|
model="mock", messages=[{"role": "user", "content": "hi"}], **call_kwargs
|
|
)
|
|
return counter["n"]
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("num_retries", [2, 5])
|
|
async def test_deployment_num_retries_sends_no_extra_provider_requests(self, num_retries):
|
|
"""
|
|
Deployment ``num_retries=N`` (every attempt failing) must send exactly ``N + 1``
|
|
upstream requests, not ``(N + 1) ** 2``. This is the amplification regression:
|
|
an unfixed build sends 9 (N=2) or 36 (N=5).
|
|
"""
|
|
counter = self._install_counting_upstream()
|
|
router = self._router(
|
|
f"https://amp-{num_retries}.local/v1", {"num_retries": num_retries}, num_retries=1
|
|
)
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.acompletion(model="mock", messages=[{"role": "user", "content": "hi"}])
|
|
assert counter["n"] == num_retries + 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_max_retries_does_not_nest_with_router_retries(self):
|
|
"""
|
|
A request-body ``max_retries`` must not make the provider SDK retry on top of the
|
|
Router. With deployment ``num_retries=5`` and request ``max_retries=3`` the count
|
|
stays ``6``; a build that lets either value reach the provider SDK sends 24 or 36.
|
|
"""
|
|
router = self._router("https://nest-req.local/v1", {"num_retries": 5}, num_retries=1)
|
|
assert await self._call_and_count(router, max_retries=3) == 6
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deployment_max_retries_does_not_nest_with_router_retries(self):
|
|
"""
|
|
A deployment-level ``max_retries`` is likewise never applied on top of the Router's
|
|
retries for a routed call: deployment ``num_retries=5`` plus ``max_retries=3`` still
|
|
sends exactly ``6`` upstream requests.
|
|
"""
|
|
router = self._router(
|
|
"https://nest-dep.local/v1", {"num_retries": 5, "max_retries": 3}, num_retries=1
|
|
)
|
|
assert await self._call_and_count(router) == 6
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_policy_configured_does_not_reintroduce_amplification(self):
|
|
"""
|
|
``InternalServerErrorRetries=2`` overrides the per-deployment ``num_retries=5`` for the
|
|
500s this upstream returns, and the provider SDK still must not retry on top: exactly
|
|
``3`` upstream requests, not 18.
|
|
"""
|
|
router = self._router(
|
|
"https://policy.local/v1",
|
|
{"num_retries": 5},
|
|
num_retries=1,
|
|
retry_policy=RetryPolicy(InternalServerErrorRetries=2),
|
|
)
|
|
assert await self._call_and_count(router) == 3
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_global_num_retries_not_amplified(self):
|
|
"""
|
|
Global ``num_retries`` (no per-deployment setting) already behaves correctly and
|
|
must stay that way: ``num_retries=3`` sends ``4`` upstream requests.
|
|
"""
|
|
router = self._router("https://global.local/v1", {}, num_retries=3)
|
|
assert await self._call_and_count(router) == 4
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_direct_completion_still_forwards_num_retries_to_provider(self):
|
|
"""
|
|
For a NON-routed direct ``litellm.acompletion`` call, ``num_retries`` remains an
|
|
alias for the provider client's ``max_retries`` (the instructor use case). The
|
|
provider SDK therefore retries in addition to litellm's own retry wrapper, so the
|
|
upstream count exceeds ``num_retries + 1`` - proving the routed-call fix did not
|
|
change direct-call behaviour.
|
|
"""
|
|
counter = self._install_counting_upstream()
|
|
num_retries = 2
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await litellm.acompletion(
|
|
model="openai/gpt-4o-mini",
|
|
api_base="https://direct.local/v1",
|
|
api_key="sk-fake",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
num_retries=num_retries,
|
|
)
|
|
assert counter["n"] > num_retries + 1
|
|
|
|
|
|
class _AttemptCounter(CustomLogger):
|
|
"""Counts upstream call attempts via the pre-call hook (one per attempt)."""
|
|
|
|
def __init__(self):
|
|
self.attempts = 0
|
|
|
|
def log_pre_api_call(self, model, messages, kwargs):
|
|
self.attempts += 1
|
|
|
|
|
|
class TestRequestNumRetriesBeatsGlobal:
|
|
"""
|
|
A per-request num_retries (request body or the x-litellm-num-retries header, both of
|
|
which arrive as the num_retries kwarg) must take precedence over the global
|
|
litellm.num_retries (litellm_settings.num_retries on the proxy) during retry handling.
|
|
|
|
The regression: the @client wrapper stamped the global litellm.num_retries onto the
|
|
raised exception, and async_function_with_retries then adopted that stamped value,
|
|
overwriting the request-level num_retries it had already resolved. This exercises the
|
|
real retry loop end to end (the failing call flows through the wrapped litellm.acompletion),
|
|
which the kwargs-merge-only test above does not.
|
|
"""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_litellm_globals(self):
|
|
prev_num_retries = litellm.num_retries
|
|
prev_callbacks = litellm.callbacks
|
|
yield
|
|
litellm.num_retries = prev_num_retries
|
|
litellm.callbacks = prev_callbacks
|
|
|
|
@staticmethod
|
|
def _router(global_num_retries):
|
|
return Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "mock",
|
|
"litellm_params": {
|
|
"model": "openai/mock",
|
|
"api_key": "sk-fake",
|
|
"mock_response": "litellm.InternalServerError",
|
|
},
|
|
}
|
|
],
|
|
num_retries=global_num_retries,
|
|
)
|
|
|
|
async def _count_attempts(self, *, global_num_retries, request_num_retries):
|
|
counter = _AttemptCounter()
|
|
litellm.callbacks = [counter]
|
|
litellm.num_retries = global_num_retries
|
|
router = self._router(global_num_retries)
|
|
kwargs = {"model": "mock", "messages": [{"role": "user", "content": "hi"}]}
|
|
if request_num_retries is not None:
|
|
kwargs["num_retries"] = request_num_retries
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.acompletion(**kwargs)
|
|
return counter.attempts
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_num_retries_overrides_global(self):
|
|
"""global=3 + request=1 -> 2 attempts (1 initial + 1 retry), not 4 (1 + global 3)."""
|
|
attempts = await self._count_attempts(global_num_retries=3, request_num_retries=1)
|
|
assert attempts == 2
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_num_retries_zero_disables_retries_despite_global(self):
|
|
"""global=3 + request=0 -> a single attempt (retries disabled by the request)."""
|
|
attempts = await self._count_attempts(global_num_retries=3, request_num_retries=0)
|
|
assert attempts == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_num_retries_zero_disables_retry_policy(self):
|
|
"""An explicit zero remains a single attempt when a retry policy matches the error."""
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "mock",
|
|
"litellm_params": {
|
|
"model": "openai/mock-timeout",
|
|
"api_key": "sk-fake",
|
|
"mock_timeout": True,
|
|
},
|
|
}
|
|
],
|
|
num_retries=3,
|
|
retry_after=0,
|
|
retry_policy=RetryPolicy(TimeoutErrorRetries=2),
|
|
)
|
|
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.Timeout):
|
|
await router.acompletion(
|
|
model="mock",
|
|
messages=[{"role": "user", "content": "hi"}],
|
|
timeout=0.001,
|
|
num_retries=0,
|
|
)
|
|
|
|
assert router.total_calls["openai/mock-timeout"] == 1
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_global_num_retries_applies_when_request_omits_it(self):
|
|
"""No request num_retries -> the global still applies: 1 initial + 3 retries = 4."""
|
|
attempts = await self._count_attempts(global_num_retries=3, request_num_retries=None)
|
|
assert attempts == 4
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deployment_num_retries_reaches_wrapper_when_no_request_value(self):
|
|
"""
|
|
With no request value and the router default at 0, a deployment's
|
|
litellm_params.num_retries reaches the wrapped call, is carried on the raised
|
|
exception, and is applied: deployment 2 -> 1 initial + 2 retries = 3 (not 1).
|
|
"""
|
|
counter = _AttemptCounter()
|
|
litellm.callbacks = [counter]
|
|
litellm.num_retries = None
|
|
router = Router(
|
|
model_list=[
|
|
{
|
|
"model_name": "mock",
|
|
"litellm_params": {
|
|
"model": "openai/mock",
|
|
"api_key": "sk-fake",
|
|
"mock_response": "litellm.InternalServerError",
|
|
"num_retries": 2,
|
|
},
|
|
}
|
|
],
|
|
num_retries=0,
|
|
)
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.acompletion(
|
|
model="mock", messages=[{"role": "user", "content": "hi"}]
|
|
)
|
|
assert counter.attempts == 3
|
|
|
|
|
|
class TestRequestNumRetriesBeatsDeployment:
|
|
"""
|
|
Documented precedence for num_retries on the proxy:
|
|
|
|
x-litellm-num-retries header > request body > model_list litellm_params > litellm_settings
|
|
|
|
The header and the body both arrive at the router as the num_retries kwarg (the proxy
|
|
overwrites the body value with the header one), so "request level" covers both.
|
|
|
|
The regression: a failing deployment stamps its own litellm_params.num_retries onto the
|
|
raised exception, and async_function_with_retries adopted that value unconditionally, so a
|
|
deployment setting outranked the header and the body instead of sitting below them.
|
|
"""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_litellm_globals(self):
|
|
prev_num_retries = litellm.num_retries
|
|
prev_callbacks = litellm.callbacks
|
|
yield
|
|
litellm.num_retries = prev_num_retries
|
|
litellm.callbacks = prev_callbacks
|
|
|
|
@staticmethod
|
|
def _router(*, global_num_retries, deployment_num_retries):
|
|
litellm_params = {
|
|
"model": "openai/mock",
|
|
"api_key": "sk-fake",
|
|
"mock_response": "litellm.InternalServerError",
|
|
}
|
|
if deployment_num_retries is not None:
|
|
litellm_params["num_retries"] = deployment_num_retries
|
|
return Router(
|
|
model_list=[{"model_name": "mock", "litellm_params": litellm_params}],
|
|
num_retries=global_num_retries,
|
|
)
|
|
|
|
async def _count_attempts(
|
|
self,
|
|
*,
|
|
global_num_retries,
|
|
deployment_num_retries,
|
|
request_num_retries,
|
|
mock_testing_rate_limit_error=False,
|
|
):
|
|
counter = _AttemptCounter()
|
|
litellm.callbacks = [counter]
|
|
litellm.num_retries = global_num_retries
|
|
router = self._router(
|
|
global_num_retries=global_num_retries,
|
|
deployment_num_retries=deployment_num_retries,
|
|
)
|
|
kwargs = {"model": "mock", "messages": [{"role": "user", "content": "hi"}]}
|
|
if request_num_retries is not None:
|
|
kwargs["num_retries"] = request_num_retries
|
|
if mock_testing_rate_limit_error:
|
|
kwargs["mock_testing_rate_limit_error"] = True
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises((litellm.InternalServerError, litellm.RateLimitError)):
|
|
await router.acompletion(**kwargs)
|
|
return counter.attempts
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("request_num_retries, expected_attempts", [(3, 4), (0, 1)])
|
|
async def test_request_num_retries_overrides_deployment(
|
|
self, request_num_retries, expected_attempts
|
|
):
|
|
"""
|
|
global=1, deployment=2, request=3 -> 4 attempts, and request=0 -> a single attempt.
|
|
Before the fix the deployment value won, so both cases sent 3 attempts.
|
|
"""
|
|
attempts = await self._count_attempts(
|
|
global_num_retries=1,
|
|
deployment_num_retries=2,
|
|
request_num_retries=request_num_retries,
|
|
)
|
|
assert attempts == expected_attempts
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deployment_num_retries_still_beats_global_when_request_omits_it(self):
|
|
"""
|
|
The deployment value keeps its place directly above litellm_settings: global=1 and
|
|
deployment=2 with no request value -> 1 initial attempt + 2 retries.
|
|
"""
|
|
attempts = await self._count_attempts(
|
|
global_num_retries=1, deployment_num_retries=2, request_num_retries=None
|
|
)
|
|
assert attempts == 3
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_global_num_retries_applies_when_neither_request_nor_deployment_sets_it(self):
|
|
"""Bottom of the chain is unchanged: global=3, nothing else set -> 4 attempts."""
|
|
attempts = await self._count_attempts(
|
|
global_num_retries=3, deployment_num_retries=None, request_num_retries=None
|
|
)
|
|
assert attempts == 4
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_num_retries_overrides_deployment_on_the_rate_limit_mock_path(self):
|
|
"""
|
|
mock_testing_rate_limit_error raises before the first upstream call and carries the
|
|
deployment's num_retries on the mock exception - a second place the deployment value is
|
|
injected. The request value must still win: request=3 leaves 3 real attempts, where the
|
|
deployment value would leave 2.
|
|
"""
|
|
attempts = await self._count_attempts(
|
|
global_num_retries=1,
|
|
deployment_num_retries=2,
|
|
request_num_retries=3,
|
|
mock_testing_rate_limit_error=True,
|
|
)
|
|
assert attempts == 3
|
|
|
|
|
|
class TestDeploymentNumRetriesOnNonCompletionEntryPoints:
|
|
"""
|
|
aimage_generation and its siblings (adapter completion, file create, batch create, batch cancel)
|
|
run through the same retry loop as acompletion, so a deployment's litellm_params.num_retries must
|
|
apply there too whenever the request carries none.
|
|
|
|
These entry points used to pre-fill num_retries with the router default before handing kwargs to
|
|
the retry loop. That default is indistinguishable from a caller-supplied value, so it would
|
|
permanently suppress the deployment setting once the loop started ranking request above
|
|
deployment.
|
|
|
|
The provider SDK adds a fixed number of its own attempts per router attempt on this path, so the
|
|
factor is calibrated from a single-router-attempt run rather than hard coded.
|
|
"""
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _isolate(self):
|
|
prev = litellm.num_retries
|
|
litellm.num_retries = None
|
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
|
yield
|
|
litellm.num_retries = prev
|
|
litellm.aclient_session = None
|
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
|
|
|
async def _upstream_count(self, label, deployment_num_retries=None, **call_kwargs):
|
|
counter = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
counter["n"] += 1
|
|
return httpx.Response(500, headers={"retry-after": "0"}, json={"error": {"message": "boom"}})
|
|
|
|
litellm.aclient_session = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
litellm.in_memory_llm_clients_cache.flush_cache()
|
|
litellm_params = {
|
|
"model": "openai/dall-e-3",
|
|
"api_key": "sk-fake",
|
|
"api_base": f"https://imgretry-{label}.local/v1",
|
|
}
|
|
if deployment_num_retries is not None:
|
|
litellm_params["num_retries"] = deployment_num_retries
|
|
router = Router(
|
|
model_list=[{"model_name": "img", "litellm_params": litellm_params}],
|
|
num_retries=0,
|
|
)
|
|
with patch("asyncio.sleep", return_value=None):
|
|
with pytest.raises(litellm.InternalServerError):
|
|
await router.aimage_generation(model="img", prompt="a cat", **call_kwargs)
|
|
return counter["n"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deployment_num_retries_applies_to_image_generation(self):
|
|
"""
|
|
Router default 0, deployment 2, no request value -> 3 router attempts. Asserted against a
|
|
calibrated single-attempt run so the provider SDK's own attempt count does not matter.
|
|
"""
|
|
one_attempt = await self._upstream_count("calibrate")
|
|
assert one_attempt > 0
|
|
assert await self._upstream_count("dep2", deployment_num_retries=2) == 3 * one_attempt
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_request_num_retries_still_wins_on_image_generation(self):
|
|
"""A request value outranks the deployment here too: deployment 4, request 1 -> 2 attempts."""
|
|
one_attempt = await self._upstream_count("calibrate2")
|
|
assert await self._upstream_count("req1", deployment_num_retries=4, num_retries=1) == 2 * one_attempt
|